feat: add dev config with support for prom loki and tempo (#5647)

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

Makefile

@@ -30,6 +30,7 @@ REPO_KYVERNOPRE      := $(REGISTRY)/$(REPO)/$(KYVERNOPRE_IMAGE)
 REPO_KYVERNO         := $(REGISTRY)/$(REPO)/$(KYVERNO_IMAGE)
 REPO_CLI             := $(REGISTRY)/$(REPO)/$(CLI_IMAGE)
 REPO_CLEANUP         := $(REGISTRY)/$(REPO)/$(KYVERNO_CLEANUP)
+USE_CONFIG           ?= standard
 
 #########
 # TOOLS #
@@ -697,7 +698,7 @@ test-perf: $(PACKAGE_SHIM)
 .PHONY: kind-create-cluster
 kind-create-cluster: $(KIND) ## Create kind cluster
 	@echo Create kind cluster... >&2
-	@$(KIND) create cluster --name $(KIND_NAME) --image $(KIND_IMAGE) --config ./scripts/kind.yaml
+	@$(KIND) create cluster --name $(KIND_NAME) --image $(KIND_IMAGE) --config ./scripts/config/kind.yaml
 
 .PHONY: kind-delete-cluster
 kind-delete-cluster: $(KIND) ## Delete kind cluster
@@ -725,45 +726,76 @@ kind-load-all: kind-load-kyvernopre kind-load-kyverno kind-load-cleanup-controll
 .PHONY: kind-deploy-kyverno
 kind-deploy-kyverno: $(HELM) kind-load-all ## Build images, load them in kind cluster and deploy kyverno helm chart
 	@echo Install kyverno chart... >&2
-	@$(HELM) upgrade --install kyverno --namespace kyverno --wait --create-namespace ./charts/kyverno \
+	@$(HELM) upgrade --install kyverno --namespace kyverno --create-namespace --wait ./charts/kyverno \
 		--set cleanupController.image.repository=$(LOCAL_CLEANUP_IMAGE) \
 		--set cleanupController.image.tag=$(IMAGE_TAG_DEV) \
 		--set image.repository=$(LOCAL_KYVERNO_IMAGE) \
 		--set image.tag=$(IMAGE_TAG_DEV) \
 		--set initImage.repository=$(LOCAL_KYVERNOPRE_IMAGE) \
 		--set initImage.tag=$(IMAGE_TAG_DEV) \
-		--values ./scripts/kyverno.yaml
+		--values ./scripts/config/$(USE_CONFIG)/kyverno.yaml
 	@echo Restart kyverno pods... >&2
 	@kubectl rollout restart deployment -n kyverno
 
 .PHONY: kind-deploy-kyverno-policies
 kind-deploy-kyverno-policies: $(HELM) ## Deploy kyverno-policies helm chart
 	@echo Install kyverno-policies chart... >&2
-	@$(HELM) upgrade --install kyverno-policies --namespace kyverno --wait --create-namespace ./charts/kyverno-policies
+	@$(HELM) upgrade --install kyverno-policies --namespace kyverno --create-namespace --wait ./charts/kyverno-policies \
-
+		--values ./scripts/config/$(USE_CONFIG)/kyverno-policies.yaml
-.PHONY: kind-deploy-metrics-server
-kind-deploy-metrics-server: $(HELM) ## Deploy metrics-server helm chart
-	@echo Install metrics-server chart... >&2
-	@$(HELM) upgrade --install metrics-server --namespace kube-system --wait --repo https://charts.bitnami.com/bitnami metrics-server \
-		--set extraArgs={--kubelet-insecure-tls=true} \
-		--set apiService.create=true
 
 .PHONY: kind-deploy-all
-kind-deploy-all: kind-deploy-metrics-server | kind-deploy-kyverno kind-deploy-kyverno-policies ## Build images, load them in kind cluster and deploy helm charts
+kind-deploy-all: | kind-deploy-kyverno kind-deploy-kyverno-policies ## Build images, load them in kind cluster and deploy helm charts
 
 .PHONY: kind-deploy-reporter
 kind-deploy-reporter: $(HELM) ## Deploy policy-reporter helm chart
 	@echo Install policy-reporter chart... >&2
-	@$(HELM) upgrade --install policy-reporter --namespace policy-reporter --wait --repo https://kyverno.github.io/policy-reporter policy-reporter \
+	@$(HELM) upgrade --install policy-reporter --namespace policy-reporter --create-namespace --wait \
-		--set ui.enabled=true \
+		--repo https://kyverno.github.io/policy-reporter policy-reporter \
-		--set kyvernoPlugin.enabled=true \
+		--values ./scripts/config/standard/kyverno-reporter.yaml
-		--create-namespace
 	@kubectl port-forward -n policy-reporter services/policy-reporter-ui  8082:8080
 
-deploy-kube-prom-stack: $(HELM)
+###########
+# DEV LAB #
+###########
+
+.PHONY: dev-lab-ingress-ngingx
+dev-lab-ingress-ngingx: ## Deploy ingress-ngingx
+	@echo Install ingress-ngingx... >&2
+	@kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
+	@sleep 15
+	@kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=90s
+
+.PHONY: dev-lab-prometheus
+dev-lab-prometheus: $(HELM) ## Deploy kube-prometheus-stack helm chart
+	@echo Install kube-prometheus-stack chart... >&2
 	@$(HELM) upgrade --install kube-prometheus-stack --namespace monitoring --create-namespace --wait \
 		--repo https://prometheus-community.github.io/helm-charts kube-prometheus-stack \
-		--values ./scripts/kube-prometheus-stack.yaml
+		--values ./scripts/config/dev/kube-prometheus-stack.yaml
+
+.PHONY: dev-lab-loki
+dev-lab-loki: $(HELM) ## Deploy loki-stack helm chart
+	@echo Install loki-stack chart... >&2
+	@$(HELM) upgrade --install loki-stack --namespace monitoring --create-namespace --wait \
+		--repo https://grafana.github.io/helm-charts loki-stack \
+		--values ./scripts/config/dev/loki-stack.yaml
+
+.PHONY: dev-lab-tempo
+dev-lab-tempo: $(HELM) ## Deploy tempo helm chart
+	@echo Install tempo chart... >&2
+	@$(HELM) upgrade --install tempo --namespace monitoring --create-namespace --wait \
+		--repo https://grafana.github.io/helm-charts tempo \
+		--values ./scripts/config/dev/tempo.yaml
+	@kubectl apply -f ./scripts/config/dev/tempo-datasource.yaml
+
+.PHONY: dev-lab-metrics-server
+dev-lab-metrics-server: $(HELM) ## Deploy metrics-server helm chart
+	@echo Install metrics-server chart... >&2
+	@$(HELM) upgrade --install metrics-server --namespace kube-system --wait \
+		--repo https://charts.bitnami.com/bitnami metrics-server \
+		--values ./scripts/config/dev/metrics-server.yaml
+
+.PHONY: dev-lab-all
+dev-lab-all: dev-lab-ingress-ngingx dev-lab-metrics-server dev-lab-prometheus dev-lab-loki dev-lab-tempo
 
 ########
 # HELP #

scripts/config/dev/kube-prometheus-stack.yaml

@@ -2,8 +2,10 @@ kubeEtcd:
   service:
     enabled: true
     targetPort: 2381
+
 defaultRules:
   create: true
+
 alertmanager:
   alertmanagerSpec:
     routePrefix: /alertmanager
@@ -14,6 +16,7 @@ alertmanager:
   ingress:
     enabled: true
     pathType: Prefix
+
 prometheus:
   prometheusSpec:
     externalUrl: /prometheus
@@ -25,6 +28,7 @@ prometheus:
   ingress:
     enabled: true
     pathType: Prefix
+
 grafana:
   enabled: true
   adminPassword: admin
@@ -45,30 +49,3 @@ grafana:
   ingress:
     enabled: true
     path: /grafana
-
-
-# kubectl apply -f - <<EOF
-# apiVersion: argoproj.io/v1alpha1
-# kind: Application
-# metadata:
-#   name: kube-prometheus-stack
-#   namespace: argocd
-# spec:
-#   destination:
-#     namespace: monitoring
-#     server: https://kubernetes.default.svc
-#   project: default
-#   source:
-#     chart: kube-prometheus-stack
-#     repoURL: https://prometheus-community.github.io/helm-charts
-#     targetRevision: 41.4.1
-#     helm:
-#       values: |
-#   syncPolicy:
-#     automated:
-#       prune: true
-#       selfHeal: true
-#     syncOptions:
-#       - CreateNamespace=true
-#       - Replace=true
-# EOF

scripts/config/dev/kyverno-policies.yaml

@@ -0,0 +1 @@
+validationFailureAction: Enforce

scripts/config/dev/kyverno.yaml

@@ -0,0 +1,24 @@
+initContainer:
+  extraArgs:
+    - --loggingFormat=json
+
+extraArgs:
+  - --loggingFormat=json
+  - --enableTracing
+  - --tracingAddress=tempo.monitoring
+  - --tracingPort=4317
+
+serviceMonitor:
+  enabled: true
+
+grafana:
+  enabled: true
+
+cleanupController:
+  rbac:
+    clusterRole:
+      extraResources:
+        - apiGroups:
+            - ''
+          resources:
+            - pods

scripts/config/dev/loki-stack.yaml

@@ -0,0 +1,2 @@
+loki:
+  isDefault: false

scripts/config/dev/metrics-server.yaml

@@ -0,0 +1,5 @@
+extraArgs:
+  - --kubelet-insecure-tls=true
+
+apiService:
+  create: true

scripts/config/dev/tempo-datasource.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    grafana_datasource: "1"
+  name: tempo-datasource
+  namespace: monitoring
+data:
+  tempo-datasource.yaml: |-
+    apiVersion: 1
+    datasources:
+    - name: Tempo
+      type: tempo
+      access: proxy
+      url: "http://tempo.monitoring:3100"
+      version: 1
+      isDefault: false

scripts/config/dev/tempo.yaml

@@ -0,0 +1,2 @@
+tempo:
+  searchEnabled: true

scripts/config/standard/kyverno-policies.yaml

@@ -0,0 +1 @@
+validationFailureAction: Audit

scripts/config/standard/kyverno.yaml

@@ -1,9 +1,9 @@
 initContainer:
   extraArgs:
-    - --loggingFormat=text
+    - --loggingFormat=json
 
 extraArgs:
-  - --loggingFormat=text
+  - --loggingFormat=json
 
 cleanupController:
   rbac:
@@ -13,4 +13,3 @@ cleanupController:
             - ''
           resources:
             - pods
-

scripts/config/standard/policy-reporter.yaml

@@ -0,0 +1,5 @@
+ui:
+  enabled: true
+
+kyvernoPlugin:
+  enabled: true