chore: add performance tests tool (#5241)

* feat: add flag to configure the number of background scan workers

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* feat: add performance testing

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix pvc issue

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* make nodes count configurable

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* kube proxy

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* remove commented code

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* memory request

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>
Co-authored-by: shuting <shuting@nirmata.com>

Makefile

@@ -640,6 +640,38 @@ gh-install-pin-github-action:
 gh-pin-actions: gh-install-pin-github-action
 	@pin-github-action ./.github/workflows/release.yaml
 
+#############
+# PERF TEST #
+#############
+
+PERF_TEST_NODE_COUNT		?= 3
+PERF_TEST_MEMORY_REQUEST	?= "1Gi"
+
+.PHONY: test-perf
+test-perf: $(PACKAGE_SHIM)
+	GO111MODULE=off GOPATH=$(GOPATH_SHIM) go get k8s.io/perf-tests || true
+	cd $(GOPATH_SHIM)/src/k8s.io/perf-tests && \
+	GOPATH=$(GOPATH_SHIM) ./run-e2e.sh cluster-loader2 \
+		--testconfig=./testing/load/config.yaml \
+		--provider=kind \
+		--kubeconfig=${HOME}/.kube/config \
+		--nodes=$(PERF_TEST_NODE_COUNT) \
+		--prometheus-memory-request=$(PERF_TEST_MEMORY_REQUEST) \
+		--enable-prometheus-server=true \
+		--tear-down-prometheus-server=true \
+		--prometheus-apiserver-scrape-port=6443 \
+		--prometheus-scrape-kubelets=true \
+		--prometheus-scrape-master-kubelets=true \
+		--prometheus-scrape-etcd=true \
+		--prometheus-scrape-kube-proxy=true \
+		--prometheus-kube-proxy-selector-key=k8s-app \
+		--prometheus-scrape-node-exporter=false \
+		--prometheus-scrape-kube-state-metrics=true \
+		--prometheus-scrape-metrics-server=true \
+		--prometheus-pvc-storage-class=standard \
+		--v=2 \
+		--report-dir=.
+
 ########
 # KIND #
 ########
@@ -690,12 +722,12 @@ kind-deploy-kyverno: $(HELM) kind-load-all ## Build images, load them in kind cl
 .PHONY: kind-deploy-kyverno-policies
 kind-deploy-kyverno-policies: $(HELM) ## Deploy kyverno-policies helm chart
 	@echo Install kyverno-policies chart... >&2
-	@$(HELM) upgrade --install kyverno-policies --namespace kyverno --create-namespace ./charts/kyverno-policies
+	@$(HELM) upgrade --install kyverno-policies --namespace kyverno --wait --create-namespace ./charts/kyverno-policies
 
 .PHONY: kind-deploy-metrics-server
 kind-deploy-metrics-server: $(HELM) ## Deploy metrics-server helm chart
 	@echo Install metrics-server chart... >&2
-	@$(HELM) upgrade --install metrics-server --repo https://charts.bitnami.com/bitnami metrics-server -n kube-system \
+	@$(HELM) upgrade --install metrics-server --namespace kube-system --wait --repo https://charts.bitnami.com/bitnami metrics-server \
 		--set extraArgs={--kubelet-insecure-tls=true} \
 		--set apiService.create=true
 
@@ -705,12 +737,17 @@ kind-deploy-all: kind-deploy-metrics-server | kind-deploy-kyverno kind-deploy-ky
 .PHONY: kind-deploy-reporter
 kind-deploy-reporter: $(HELM) ## Deploy policy-reporter helm chart
 	@echo Install policy-reporter chart... >&2
-	@$(HELM) upgrade --install policy-reporter --repo https://kyverno.github.io/policy-reporter policy-reporter -n policy-reporter \
+	@$(HELM) upgrade --install policy-reporter --namespace policy-reporter --wait --repo https://kyverno.github.io/policy-reporter policy-reporter \
 		--set ui.enabled=true \
 		--set kyvernoPlugin.enabled=true \
 		--create-namespace
 	@kubectl port-forward -n policy-reporter services/policy-reporter-ui  8082:8080
 
+deploy-kube-prom-stack: $(HELM)
+	@$(HELM) upgrade --install kube-prometheus-stack --namespace monitoring --create-namespace --wait \
+		--repo https://prometheus-community.github.io/helm-charts kube-prometheus-stack \
+		--values ./scripts/kube-prometheus-stack.yaml
+
 ########
 # HELP #
 ########

scripts/kind.yaml

@@ -9,7 +9,7 @@ kubeadmConfigPatches:
     etcd:
       local:
         extraArgs:
-          listen-metrics-urls: http://0.0.0.0:2381
+          listen-metrics-urls: http://0.0.0.0:2382
     scheduler:
       extraArgs:
         bind-address: 0.0.0.0

scripts/kube-prometheus-stack.yaml

@@ -0,0 +1,74 @@
+kubeEtcd:
+  service:
+    enabled: true
+    targetPort: 2381
+defaultRules:
+  create: true
+alertmanager:
+  alertmanagerSpec:
+    routePrefix: /alertmanager
+    alertmanagerConfigSelector:
+      matchLabels: {}
+    alertmanagerConfigNamespaceSelector:
+      matchLabels: {}
+  ingress:
+    enabled: true
+    pathType: Prefix
+prometheus:
+  prometheusSpec:
+    externalUrl: /prometheus
+    routePrefix: /prometheus
+    ruleSelectorNilUsesHelmValues: false
+    serviceMonitorSelectorNilUsesHelmValues: false
+    podMonitorSelectorNilUsesHelmValues: false
+    probeSelectorNilUsesHelmValues: false
+  ingress:
+    enabled: true
+    pathType: Prefix
+grafana:
+  enabled: true
+  adminPassword: admin
+  sidecar:
+    enableUniqueFilenames: true
+    dashboards:
+      enabled: true
+      searchNamespace: ALL
+      provider:
+        foldersFromFilesStructure: true
+    datasources:
+      enabled: true
+      searchNamespace: ALL
+  grafana.ini:
+    server:
+      root_url: "%(protocol)s://%(domain)s:%(http_port)s/grafana"
+      serve_from_sub_path: true
+  ingress:
+    enabled: true
+    path: /grafana
+
+
+# kubectl apply -f - <<EOF
+# apiVersion: argoproj.io/v1alpha1
+# kind: Application
+# metadata:
+#   name: kube-prometheus-stack
+#   namespace: argocd
+# spec:
+#   destination:
+#     namespace: monitoring
+#     server: https://kubernetes.default.svc
+#   project: default
+#   source:
+#     chart: kube-prometheus-stack
+#     repoURL: https://prometheus-community.github.io/helm-charts
+#     targetRevision: 41.4.1
+#     helm:
+#       values: |
+#   syncPolicy:
+#     automated:
+#       prune: true
+#       selfHeal: true
+#     syncOptions:
+#       - CreateNamespace=true
+#       - Replace=true
+# EOF