feat(ci): enhance load testing (#11429)

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>
Co-authored-by: shuting <shuting@nirmata.com>

.github/workflows/load-testing.yml

@@ -1,18 +1,33 @@
-name: Baseline Load Tests
+name: Load Tests
 
 permissions: {}
 
 on:
+  release:
+    types: [published]
   pull_request:
     branches:
       - "main"
       - "release*"
+  schedule:
+    - cron: "27 0 * * 0"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
+  define-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      tests: ${{ steps.set-tests.outputs.tests }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Set Tests
+        id: set-tests
+        run: echo "tests=$(jq -c . < ./test/load/k6/${{ github.event_name }}-matrix.json)" >> $GITHUB_OUTPUT
+
   prepare-images:
     runs-on: ubuntu-latest
     steps:
@@ -42,34 +57,19 @@ jobs:
           retention-days: 1
           if-no-files-found: error
 
-  load-test:
+  old-load-test:
+    if: github.event_name == 'pull_request'
+    needs:
+      - prepare-images
+    outputs:
+      p95: ${{ steps.extract-p95.outputs.p95 }}
     runs-on: ubuntu-latest
     permissions:
       packages: read
     strategy:
       fail-fast: false
       matrix:
-        k8s-version:
+        k8s-version: [v1.31.0]
-          - name: v1.30
-            version: v1.31.0
-        kyverno-config:
-          - name: default
-            values:
-              - default-with-profiling
-          - name: stress
-            values:
-              - stress-with-profiling
-        test:
-          - kyverno-pss
-          - kyverno-mutate
-        k6-config:
-          - vus: 5
-            iterations: 100
-          - vus: 10
-            iterations: 200
-    needs:
-      - prepare-images
-    name: ${{ matrix.kyverno-config.name }} - ${{ matrix.test }} - ${{ matrix.k6-config.vus }} vus - ${{ matrix.k6-config.iterations }} iterations
     steps:
       - name: Checkout kyverno/kyverno
         uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
@@ -78,65 +78,249 @@ jobs:
         with:
           repository: kyverno/load-testing
           path: load-testing
-      - name: Install helm
+      - name: Install Helm
         id: helm
         uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
-      - name: Create kind cluster
+      - name: Create Kind cluster
         uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
         with:
-          node_image: kindest/node:${{ matrix.k8s-version.version }}
+          node_image: kindest/node:${{ matrix.k8s-version }}
           cluster_name: kind
           config: ./scripts/config/kind/default.yaml
       - name: Download kyverno images archive
         uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           name: kyverno.tar
-      - name: Load kyverno images archive in kind cluster
+      - name: Load Kyverno images archive in Kind cluster
         shell: bash
         run: |
           set -e
           kind load image-archive kyverno.tar --name kind
-      - name: Install kyverno
+      - name: Install Kyverno
         shell: bash
         run: |
           set -e
           export HELM=${{ steps.helm.outputs.helm-path }}
-          export USE_CONFIG=${{ join(matrix.kyverno-config.values, ',') }}
+          export USE_CONFIG=default-with-profiling
-          make kind-install-kyverno
+          $HELM repo add kyverno https://kyverno.github.io/kyverno/
-          bash load-testing/k8s/metrics-server/hack.sh
+          $HELM repo update
-          make kind-install-goldilocks
+          export INSTALL_VERSION=$($HELM search repo kyverno/kyverno -o json | jq -r '.[0].version')
+          export EXPLICIT_INSTALL_SETTINGS='--set admissionController.replicas=1 --set admissionController.resources.requests.cpu=100m --set admissionController.resources.limits.cpu=1500m --set admissionController.resources.requests.memory=128Mi --set admissionController.resources.limits.memory=384Mi'
+          make kind-install-kyverno-from-repo
       - name: Wait for kyverno ready
         uses: ./.github/actions/kyverno-wait-ready
+      - name: Install K6
+        shell: bash
+        run: |
+          set -e
+          go install go.k6.io/xk6/cmd/xk6@latest
+          $(go env GOPATH)/bin/xk6 build --with github.com/grafana/xk6-dashboard@latest
+          mkdir -p $HOME/.local/bin && mv ./k6 $HOME/.local/bin
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
       - name: Run load tests using K6
         shell: bash
         run: |
           set -e
+          mkdir -p report
           KYVERNO_NODE_IP=$(kubectl get nodes -o jsonpath='{.items[?(@.metadata.labels.kubernetes\.io/hostname=="kind-control-plane")].status.addresses[?(@.type=="InternalIP")].address}')
-          curl http://$KYVERNO_NODE_IP:30950/debug/pprof/heap > heap.pprof
+          curl -s "http://$KYVERNO_NODE_IP:30950/debug/pprof/profile?seconds=90" > report/cpu.pprof &
-          curl "http://$KYVERNO_NODE_IP:30950/debug/pprof/profile?seconds=30" > cpu.pprof 2> curl.tmp &
+          cd load-testing
-          cd load-testing/k6
+          ./k6/run.sh k6/tests/kyverno-pss.js -e SCENARIO=average --out dashboard=export=load-report.html
-          ./start.sh tests/${{ matrix.test }}.js ${{ matrix.k6-config.vus }} ${{ matrix.k6-config.iterations }}
           wait %1 || true
-          # TODO: wait for VPA to stabilize and recommend
+          mv load-report.html ../report
-          kubectl -n kyverno get vpa goldilocks-kyverno-admission-controller -o jsonpath='{.status.recommendation.containerRecommendations[*]}'
+      - name: Extract P(95)
-      - name: Archive load test results
+        id: extract-p95
-        if: failure()
+        shell: bash
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        run: |
-        with:
+          set -e
-          name: load-test-reports
+          echo "p95=$(grep http_req_duration load-testing/test-output.log | awk -F 'p\\(95\\)=' '{split($2,a,\"ms\"); print a[1]}')" >> $GITHUB_OUTPUT
-          path: load-testing/k6/${{ matrix.test }}.js-${{ matrix.k6-config.vus }}vu-${{ matrix.k6-config.iterations }}it-logs.txt
+          echo $GITHUB_OUTPUT
-      - name: Archive pprof CPU profiles
-        if: failure()
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
-        with:
-          name: pprof-cpu-profiles
-          path: cpu.pprof
-      - name: Archive pprof HEAP profiles
-        if: failure()
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
-        with:
-          name: pprof-heap-profiles
-          path: heap.pprof
       - name: Debug failure
         if: failure()
         uses: ./.github/actions/kyverno-logs
+
+  load-test:
+    if: github.event_name == 'pull_request'
+    needs:
+      - prepare-images
+      - old-load-test
+    runs-on: ubuntu-latest
+    permissions:
+      packages: read
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s-version: [v1.31.0]
+    steps:
+      - name: Checkout kyverno/kyverno
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Checkout kyverno/load-testing
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          repository: kyverno/load-testing
+          path: load-testing
+      - name: Install Helm
+        id: helm
+        uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
+      - name: Create Kind cluster
+        uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+        with:
+          node_image: kindest/node:${{ matrix.k8s-version }}
+          cluster_name: kind
+          config: ./scripts/config/kind/default.yaml
+      - name: Download kyverno images archive
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: kyverno.tar
+      - name: Load Kyverno images archive in Kind cluster
+        shell: bash
+        run: |
+          set -e
+          kind load image-archive kyverno.tar --name kind
+      - name: Install Kyverno
+        shell: bash
+        run: |
+          set -e
+          export HELM=${{ steps.helm.outputs.helm-path }}
+          export USE_CONFIG=default-with-profiling
+          export EXPLICIT_INSTALL_SETTINGS='--set admissionController.replicas=1 --set admissionController.resources.requests.cpu=100m --set admissionController.resources.limits.cpu=1500m --set admissionController.resources.requests.memory=128Mi --set admissionController.resources.limits.memory=384Mi'
+          make kind-install-kyverno
+      - name: Wait for kyverno ready
+        uses: ./.github/actions/kyverno-wait-ready
+      - name: Install K6
+        shell: bash
+        run: |
+          set -e
+          go install go.k6.io/xk6/cmd/xk6@latest
+          $(go env GOPATH)/bin/xk6 build --with github.com/grafana/xk6-dashboard@latest
+          mkdir -p $HOME/.local/bin && mv ./k6 $HOME/.local/bin
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+      - name: Run load tests using K6
+        shell: bash
+        run: |
+          set -e
+          mkdir -p report
+          KYVERNO_NODE_IP=$(kubectl get nodes -o jsonpath='{.items[?(@.metadata.labels.kubernetes\.io/hostname=="kind-control-plane")].status.addresses[?(@.type=="InternalIP")].address}')
+          curl -s "http://$KYVERNO_NODE_IP:30950/debug/pprof/profile?seconds=90" > report/cpu.pprof &
+          cd load-testing
+          ./k6/run.sh k6/tests/kyverno-pss.js -e SCENARIO=average --out dashboard=export=load-report.html
+          wait %1 || true
+          mv load-report.html ../report
+      - name: Compare P(95)
+        shell: bash
+        run: |
+          set -e
+          echo "Old P(95): ${{ needs.old-load-test.outputs.p95 }}"
+          OLD_NUM=${{ needs.old-load-test.outputs.p95 }}
+          NEW_NUM=$(grep http_req_duration load-testing/test-output.log | awk -F 'p\\(95\\)=' '{split($2,a,"ms"); print a[1]}')
+          echo "$OLD_NUM to $NEW_NUM"
+          if [ $(echo "$OLD_NUM < $NEW_NUM" | bc) -eq 1 ]; then
+            echo "P(95) increased from $OLD_NUM to $NEW_NUM"
+            exit 1
+          fi
+      - name: Archive Report
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+        with:
+          name: load-test-report.html
+          path: report
+      - name: Debug failure
+        if: failure()
+        uses: ./.github/actions/kyverno-logs
+
+  scale-test:
+    if: github.event_name == 'pull_request'
+    needs:
+      - define-matrix
+      - prepare-images
+    runs-on: ubuntu-latest
+    permissions:
+      packages: read
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s-version: [v1.31.0]
+        test: ${{ fromJson(needs.define-matrix.outputs.tests) }}
+    steps:
+      - name: Checkout kyverno/kyverno
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Checkout kyverno/load-testing
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          repository: kyverno/load-testing
+          path: load-testing
+      - name: Install Helm
+        id: helm
+        uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
+      - name: Create Kind cluster
+        uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+        with:
+          node_image: kindest/node:${{ matrix.k8s-version }}
+          cluster_name: kind
+          config: ./scripts/config/kind/default.yaml
+      - name: Download kyverno images archive
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: kyverno.tar
+      - name: Load Kyverno images archive in Kind cluster
+        shell: bash
+        run: |
+          set -e
+          kind load image-archive kyverno.tar --name kind
+      - name: Install Metrics Server and Prometheus
+        shell: bash
+        run: |
+          set -e
+          export HELM=${{ steps.helm.outputs.helm-path }}
+          make dev-lab-metrics-server dev-lab-prometheus
+      - name: Install Kyverno
+        shell: bash
+        run: |
+          set -e
+          export HELM=${{ steps.helm.outputs.helm-path }}
+          export USE_CONFIG=default-with-profiling
+          export EXPLICIT_INSTALL_SETTINGS='--set admissionController.replicas=${{ matrix.test.replicas }} --set admissionController.serviceMonitor.enabled=true --set reportsController.serviceMonitor.enabled=true --set admissionController.container.resources.requests.cpu=${{ matrix.test.cpu_request }} --set admissionController.container.resources.requests.memory=${{ matrix.test.memory_request }} --set admissionController.container.resources.limits.memory=${{ matrix.test.memory_limit }} --set reportsController.resources.limits.memory=10Gi'
+          make kind-install-kyverno
+      - name: Wait for kyverno ready
+        uses: ./.github/actions/kyverno-wait-ready
+      - name: Install K6
+        shell: bash
+        run: |
+          set -e
+          go install go.k6.io/xk6/cmd/xk6@latest
+          $(go env GOPATH)/bin/xk6 build --with github.com/grafana/xk6-dashboard@latest
+          mkdir -p $HOME/.local/bin && mv ./k6 $HOME/.local/bin
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+      - name: Run load tests using K6
+        shell: bash
+        run: |
+          set -e
+          mkdir -p report
+          KYVERNO_NODE_IP=$(kubectl get nodes -o jsonpath='{.items[?(@.metadata.labels.kubernetes\.io/hostname=="kind-control-plane")].status.addresses[?(@.type=="InternalIP")].address}')
+          curl -s "http://$KYVERNO_NODE_IP:30950/debug/pprof/profile?seconds=30" > report/cpu.pprof &
+          cd load-testing
+          ./k6/run.sh k6/tests/${{ matrix.test.name }}.js -e SCENARIO=${{ matrix.test.scenario }} --vus ${{ matrix.test.concurrent_connections }} --iterations ${{ matrix.test.total_iterations }} ${{ matrix.test.extra_options }} --out dashboard=export=load-report.html
+          wait %1 || true
+          mv load-report.html ../report
+      - name: Collect Resource Metrics
+        shell: bash
+        run: |
+          set -e
+          kubectl port-forward --address 127.0.0.1 svc/kube-prometheus-stack-prometheus 9090:9090 -n monitoring &
+          sleep 3
+          curl -s "http://127.0.0.1:9090/prometheus/api/v1/query?query=$(echo -n "rate(container_cpu_usage_seconds_total{image=\"$(make kind-admission-controller-image-name)\"}[1m])" | jq -sRr @uri)" > report/cpu-usage.json
+          curl -s "http://127.0.0.1:9090/prometheus/api/v1/query?query=$(echo -n "max_over_time(container_memory_working_set_bytes{image=\"$(make kind-admission-controller-image-name)\"}[1m])/(2^20)" | jq -sRr @uri)" > report/memory-usage.json
+          kill %1 || true
+      - name: Collect Report Metrics
+        shell: bash
+        run: |
+          set -e
+          sleep 60
+          ./test/load/k6/reports-size-in-etcd.sh > report/reports-size-in-etcd.txt
+      - name: Archive Report
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+        with:
+          name: report-${{ matrix.k8s-version }}-${{ matrix.test.name }}-${{ matrix.test.scenario }}-${{ matrix.test.replicas }}-${{ matrix.test.cpu_request }}-${{ matrix.test.memory_request }}-${{ matrix.test.memory_limit }}-${{ matrix.test.concurrent_connections }}
+          path: report
+      - name: Debug failure
+        # if: failure()
+        uses: ./.github/actions/kyverno-logs

Makefile

@@ -27,6 +27,7 @@ REPO_CLEANUP         := $(REGISTRY)/$(REPO)/$(CLEANUP_IMAGE)
 REPO_REPORTS         := $(REGISTRY)/$(REPO)/$(REPORTS_IMAGE)
 REPO_BACKGROUND      := $(REGISTRY)/$(REPO)/$(BACKGROUND_IMAGE)
 USE_CONFIG           ?= standard
+INSTALL_VERSION	     ?= 3.2.6
 
 #########
 # TOOLS #
@@ -1033,7 +1034,17 @@ kind-install-kyverno: $(HELM) ## Install kyverno helm chart
 		--set crds.migration.image.registry=$(LOCAL_REGISTRY) \
 		--set crds.migration.image.repository=$(LOCAL_CLI_REPO) \
 		--set crds.migration.image.tag=$(GIT_SHA) \
-		$(foreach CONFIG,$(subst $(COMMA), ,$(USE_CONFIG)),--values ./scripts/config/$(CONFIG)/kyverno.yaml)
+		$(foreach CONFIG,$(subst $(COMMA), ,$(USE_CONFIG)),--values ./scripts/config/$(CONFIG)/kyverno.yaml) \
+		$(EXPLICIT_INSTALL_SETTINGS)
+
+.PHONY: kind-install-kyverno-from-repo
+kind-install-kyverno-from-repo: $(HELM) ## Install Kyverno Helm Chart from the Kyverno repo
+	@echo Install kyverno chart... >&2
+	@$(HELM) upgrade --install kyverno --namespace kyverno --create-namespace --wait \
+		--repo https://kyverno.github.io/kyverno/ kyverno \
+		--version $(INSTALL_VERSION) \
+		$(foreach CONFIG,$(subst $(COMMA), ,$(USE_CONFIG)),--values ./scripts/config/$(CONFIG)/kyverno.yaml) \
+		$(EXPLICIT_INSTALL_SETTINGS)
 
 .PHONY: kind-install-goldilocks
 kind-install-goldilocks: $(HELM) ## Install goldilocks helm chart
@@ -1065,6 +1076,10 @@ kind-deploy-reporter: $(HELM) ## Deploy policy-reporter helm chart
 		--values ./scripts/config/standard/kyverno-reporter.yaml
 	@kubectl port-forward -n policy-reporter services/policy-reporter-ui  8082:8080
 
+.PHONY: kind-admission-controller-image-name
+kind-admission-controller-image-name: ## Print admission controller image name
+	@echo -n $(LOCAL_REGISTRY)/$(LOCAL_KYVERNO_REPO):$(GIT_SHA)
+
 ###########
 # ROLLOUT #
 ###########

scripts/config/standard-with-profiling/kyverno.yaml

@@ -0,0 +1,67 @@
+features:
+  policyExceptions:
+    enabled: true
+  omitEvents:
+    eventTypes: []
+
+admissionController:
+  extraArgs:
+    v: 4
+  rbac:
+    clusterRole:
+      extraResources:
+        - apiGroups:
+            - "*"
+          resources:
+            - secrets
+          verbs:
+            - create
+            - update
+            - patch
+            - delete
+            - get
+            - list
+  profiling:
+    enabled: true
+    serviceType: NodePort
+    nodePort: 30950
+
+backgroundController:
+  extraArgs:
+    v: 4
+  rbac:
+    clusterRole:
+      extraResources:
+        - apiGroups:
+            - "*"
+          resources:
+            - configmaps
+            - networkpolicies
+            - resourcequotas
+            - secrets
+            - roles
+            - rolebindings
+            - limitranges
+            - namespaces
+            - nodes
+            - nodes/status
+            - pods
+          verbs:
+            - create
+            - update
+            - patch
+            - delete
+            - get
+            - list
+
+cleanupController:
+  rbac:
+    clusterRole:
+      extraResources:
+        - apiGroups:
+            - ""
+          resources:
+            - pods
+          verbs:
+            - list
+            - delete

scripts/config/stress-with-profiling/kyverno.yaml

@@ -1,19 +0,0 @@
-# stress-with-profiling sets the most restricted configurations for the admission controller. It disables
-# any additional feature that could affect the performance of the admission controller. 
-features:
-  policyExceptions:
-    enabled: true
-  omitEvents:
-    eventTypes:
-      - PolicyApplied
-      - PolicySkipped
-      - PolicyViolation
-      - PolicyError
-  admissionReports:
-    enabled: false
-
-admissionController:
-  profiling:
-    enabled: true
-    serviceType: NodePort
-    nodePort: 30950

test/load/k6/pull_request-matrix.json

@@ -0,0 +1,79 @@
+[
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 1,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 10,
+    "total_iterations": 1000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 1,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 20,
+    "total_iterations": 5000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 1,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 50,
+    "total_iterations": 10000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 3,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 10,
+    "total_iterations": 1000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 3,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 20,
+    "total_iterations": 5000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 3,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 50,
+    "total_iterations": 10000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-generate",
+    "scenario": "",
+    "replicas": 1,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 10,
+    "total_iterations": 1000,
+    "extra_options": ""
+  }
+]

test/load/k6/release-matrix.json

@@ -0,0 +1,79 @@
+[
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 1,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 10,
+    "total_iterations": 1000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 1,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 20,
+    "total_iterations": 5000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 1,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 50,
+    "total_iterations": 10000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 3,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 10,
+    "total_iterations": 1000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 3,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 20,
+    "total_iterations": 5000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 3,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 50,
+    "total_iterations": 10000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-generate",
+    "scenario": "",
+    "replicas": 1,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 10,
+    "total_iterations": 1000,
+    "extra_options": ""
+  }
+]

test/load/k6/reports-size-in-etcd.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Function to execute etcdctl commands
+execute_etcdctl() {
+    local key=$1
+    local options=$2
+    kubectl -n kube-system exec etcd-kind-control-plane -- sh -c \
+        "ETCDCTL_API=3 etcdctl --cacert /etc/kubernetes/pki/etcd/ca.crt \
+        --key /etc/kubernetes/pki/etcd/server.key \
+        --cert /etc/kubernetes/pki/etcd/server.crt \
+        get $key $options"
+}
+
+# Function to extract size and metadata
+get_key_info() {
+    local key=$1
+
+    local size=$(execute_etcdctl "$key" "--print-value-only" | wc -c)
+    local count=$(execute_etcdctl "$key" "--write-out=fields" | grep "Count" | cut -f2 -d':')
+
+    if [ "$count" -ne 0 ]; then
+        local versions=$(execute_etcdctl "$key" "--write-out=fields" | grep "Version" | cut -f2 -d':')
+    else
+        local versions=0
+    fi
+
+    # Return size, count, and versions as a string
+    echo "$size $count $versions"
+}
+
+# Initialize sum
+total_size=0
+output_file="/tmp/etcdkeys.txt"
+
+# Get list of policy report keys
+keys=$(execute_etcdctl "/registry/wgpolicyk8s.io/policyreports" "--prefix --keys-only")
+
+# Process each key
+for key in $keys; do
+    read size count versions <<< $(get_key_info "$key")
+    total=$((size * versions))
+    total_size=$((total_size + total))
+
+    # Log to output file
+    echo "$total_size $total $size $versions $count $key" >> "$output_file"
+done
+
+# Print final total size
+echo "The total size for Policy Reports is $total_size bytes."

test/load/k6/schedule-matrix.json

@@ -0,0 +1,79 @@
+[
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 1,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 10,
+    "total_iterations": 1000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 1,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 20,
+    "total_iterations": 5000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 1,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 50,
+    "total_iterations": 10000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 3,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 10,
+    "total_iterations": 1000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 3,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 20,
+    "total_iterations": 5000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-pss",
+    "scenario": "",
+    "replicas": 3,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 50,
+    "total_iterations": 10000,
+    "extra_options": "--no-teardown"
+  },
+  {
+    "name": "kyverno-generate",
+    "scenario": "",
+    "replicas": 1,
+    "cpu_request": "100m",
+    "memory_request": "128Mi",
+    "memory_limit": "384Mi",
+    "concurrent_connections": 10,
+    "total_iterations": 1000,
+    "extra_options": ""
+  }
+]