From c077fb49d506dd0c87463aaac7cb3d3f34cc0fea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Thu, 17 Nov 2022 07:31:35 +0100 Subject: [PATCH] chore: add performance tests tool (#5241) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add flag to configure the number of background scan workers Signed-off-by: Charles-Edouard Brétéché * feat: add performance testing Signed-off-by: Charles-Edouard Brétéché * fix Signed-off-by: Charles-Edouard Brétéché * fix pvc issue Signed-off-by: Charles-Edouard Brétéché * make nodes count configurable Signed-off-by: Charles-Edouard Brétéché * kube proxy Signed-off-by: Charles-Edouard Brétéché * remove commented code Signed-off-by: Charles-Edouard Brétéché * memory request Signed-off-by: Charles-Edouard Brétéché Signed-off-by: Charles-Edouard Brétéché Signed-off-by: Charles-Edouard Brétéché Co-authored-by: shuting --- Makefile | 43 +++++++++++++++-- scripts/kind.yaml | 2 +- scripts/kube-prometheus-stack.yaml | 74 ++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 scripts/kube-prometheus-stack.yaml diff --git a/Makefile b/Makefile index f4d090e058..5beefb7106 100644 --- a/Makefile +++ b/Makefile @@ -640,6 +640,38 @@ gh-install-pin-github-action: gh-pin-actions: gh-install-pin-github-action @pin-github-action ./.github/workflows/release.yaml +############# +# PERF TEST # +############# + +PERF_TEST_NODE_COUNT ?= 3 +PERF_TEST_MEMORY_REQUEST ?= "1Gi" + +.PHONY: test-perf +test-perf: $(PACKAGE_SHIM) + GO111MODULE=off GOPATH=$(GOPATH_SHIM) go get k8s.io/perf-tests || true + cd $(GOPATH_SHIM)/src/k8s.io/perf-tests && \ + GOPATH=$(GOPATH_SHIM) ./run-e2e.sh cluster-loader2 \ + --testconfig=./testing/load/config.yaml \ + --provider=kind \ + --kubeconfig=${HOME}/.kube/config \ + --nodes=$(PERF_TEST_NODE_COUNT) \ + --prometheus-memory-request=$(PERF_TEST_MEMORY_REQUEST) \ + --enable-prometheus-server=true \ + --tear-down-prometheus-server=true \ + --prometheus-apiserver-scrape-port=6443 \ + --prometheus-scrape-kubelets=true \ + --prometheus-scrape-master-kubelets=true \ + --prometheus-scrape-etcd=true \ + --prometheus-scrape-kube-proxy=true \ + --prometheus-kube-proxy-selector-key=k8s-app \ + --prometheus-scrape-node-exporter=false \ + --prometheus-scrape-kube-state-metrics=true \ + --prometheus-scrape-metrics-server=true \ + --prometheus-pvc-storage-class=standard \ + --v=2 \ + --report-dir=. + ######## # KIND # ######## @@ -690,12 +722,12 @@ kind-deploy-kyverno: $(HELM) kind-load-all ## Build images, load them in kind cl .PHONY: kind-deploy-kyverno-policies kind-deploy-kyverno-policies: $(HELM) ## Deploy kyverno-policies helm chart @echo Install kyverno-policies chart... >&2 - @$(HELM) upgrade --install kyverno-policies --namespace kyverno --create-namespace ./charts/kyverno-policies + @$(HELM) upgrade --install kyverno-policies --namespace kyverno --wait --create-namespace ./charts/kyverno-policies .PHONY: kind-deploy-metrics-server kind-deploy-metrics-server: $(HELM) ## Deploy metrics-server helm chart @echo Install metrics-server chart... >&2 - @$(HELM) upgrade --install metrics-server --repo https://charts.bitnami.com/bitnami metrics-server -n kube-system \ + @$(HELM) upgrade --install metrics-server --namespace kube-system --wait --repo https://charts.bitnami.com/bitnami metrics-server \ --set extraArgs={--kubelet-insecure-tls=true} \ --set apiService.create=true @@ -705,12 +737,17 @@ kind-deploy-all: kind-deploy-metrics-server | kind-deploy-kyverno kind-deploy-ky .PHONY: kind-deploy-reporter kind-deploy-reporter: $(HELM) ## Deploy policy-reporter helm chart @echo Install policy-reporter chart... >&2 - @$(HELM) upgrade --install policy-reporter --repo https://kyverno.github.io/policy-reporter policy-reporter -n policy-reporter \ + @$(HELM) upgrade --install policy-reporter --namespace policy-reporter --wait --repo https://kyverno.github.io/policy-reporter policy-reporter \ --set ui.enabled=true \ --set kyvernoPlugin.enabled=true \ --create-namespace @kubectl port-forward -n policy-reporter services/policy-reporter-ui 8082:8080 +deploy-kube-prom-stack: $(HELM) + @$(HELM) upgrade --install kube-prometheus-stack --namespace monitoring --create-namespace --wait \ + --repo https://prometheus-community.github.io/helm-charts kube-prometheus-stack \ + --values ./scripts/kube-prometheus-stack.yaml + ######## # HELP # ######## diff --git a/scripts/kind.yaml b/scripts/kind.yaml index 4a050d0a8b..9438061e5d 100644 --- a/scripts/kind.yaml +++ b/scripts/kind.yaml @@ -9,7 +9,7 @@ kubeadmConfigPatches: etcd: local: extraArgs: - listen-metrics-urls: http://0.0.0.0:2381 + listen-metrics-urls: http://0.0.0.0:2382 scheduler: extraArgs: bind-address: 0.0.0.0 diff --git a/scripts/kube-prometheus-stack.yaml b/scripts/kube-prometheus-stack.yaml new file mode 100644 index 0000000000..4e8b913e6b --- /dev/null +++ b/scripts/kube-prometheus-stack.yaml @@ -0,0 +1,74 @@ +kubeEtcd: + service: + enabled: true + targetPort: 2381 +defaultRules: + create: true +alertmanager: + alertmanagerSpec: + routePrefix: /alertmanager + alertmanagerConfigSelector: + matchLabels: {} + alertmanagerConfigNamespaceSelector: + matchLabels: {} + ingress: + enabled: true + pathType: Prefix +prometheus: + prometheusSpec: + externalUrl: /prometheus + routePrefix: /prometheus + ruleSelectorNilUsesHelmValues: false + serviceMonitorSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false + probeSelectorNilUsesHelmValues: false + ingress: + enabled: true + pathType: Prefix +grafana: + enabled: true + adminPassword: admin + sidecar: + enableUniqueFilenames: true + dashboards: + enabled: true + searchNamespace: ALL + provider: + foldersFromFilesStructure: true + datasources: + enabled: true + searchNamespace: ALL + grafana.ini: + server: + root_url: "%(protocol)s://%(domain)s:%(http_port)s/grafana" + serve_from_sub_path: true + ingress: + enabled: true + path: /grafana + + +# kubectl apply -f - <