kyverno/.github/workflows/load-testing.yml

name: Load Tests

permissions: {}

on:
  release:
    types: [published]
  pull_request:
    branches:
      - "main"
      - "release*"
  schedule:
    - cron: "27 0 * * 0"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  define-matrix:
    runs-on: ubuntu-latest
    outputs:
      tests: ${{ steps.set-tests.outputs.tests }}
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Set Tests
        id: set-tests
        run: echo "tests=$(jq -c . < ./test/load/k6/${{ github.event_name }}-matrix.json)" >> $GITHUB_OUTPUT

  prepare-images:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: build-images
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
        timeout-minutes: 10
        with:
          free-disk-space: false
      - name: ko build
        shell: bash
        run: |
          set -e
          VERSION=${{ github.ref_name }} make docker-save-image-all          
      - name: upload images archive
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: kyverno.tar
          path: kyverno.tar
          retention-days: 1
          if-no-files-found: error

  old-load-test:
    if: github.event_name == 'pull_request'
    needs:
      - prepare-images
    outputs:
      p95: ${{ steps.extract-p95.outputs.p95 }}
    runs-on: ubuntu-latest
    permissions:
      packages: read
    strategy:
      fail-fast: false
      matrix:
        k8s-version: [v1.31.0]
    steps:
      - name: Checkout kyverno/kyverno
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Checkout kyverno/load-testing
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: kyverno/load-testing
          path: load-testing
      - name: Install Helm
        id: helm
        uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
      - name: Create Kind cluster
        uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
        with:
          node_image: kindest/node:${{ matrix.k8s-version }}
          cluster_name: kind
          config: ./scripts/config/kind/default.yaml
      - name: Download kyverno images archive
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          name: kyverno.tar
      - name: Load Kyverno images archive in Kind cluster
        shell: bash
        run: |
          set -e
          kind load image-archive kyverno.tar --name kind          
      - name: Install Kyverno
        shell: bash
        run: |
          set -e
          export HELM=${{ steps.helm.outputs.helm-path }}
          export USE_CONFIG=default-with-profiling
          $HELM repo add kyverno https://kyverno.github.io/kyverno/
          $HELM repo update
          export INSTALL_VERSION=$($HELM search repo kyverno/kyverno -o json | jq -r '.[0].version')
          export EXPLICIT_INSTALL_SETTINGS='--set admissionController.replicas=1 --set admissionController.resources.requests.cpu=100m --set admissionController.resources.limits.cpu=1500m --set admissionController.resources.requests.memory=128Mi --set admissionController.resources.limits.memory=384Mi'
          make kind-install-kyverno-from-repo          
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Install K6
        shell: bash
        run: |
          set -e
          go install go.k6.io/xk6/cmd/xk6@latest
          $(go env GOPATH)/bin/xk6 build --with github.com/grafana/xk6-dashboard@latest
          mkdir -p $HOME/.local/bin && mv ./k6 $HOME/.local/bin
          echo "$HOME/.local/bin" >> $GITHUB_PATH          
      - name: Run load tests using K6
        shell: bash
        run: |
          set -e
          mkdir -p report
          KYVERNO_NODE_IP=$(kubectl get nodes -o jsonpath='{.items[?(@.metadata.labels.kubernetes\.io/hostname=="kind-control-plane")].status.addresses[?(@.type=="InternalIP")].address}')
          curl -s "http://$KYVERNO_NODE_IP:30950/debug/pprof/profile?seconds=90" > report/cpu.pprof &
          cd load-testing
          ./k6/run.sh k6/tests/kyverno-pss.js -e SCENARIO=average --out dashboard=export=load-report.html
          wait %1 || true
          mv load-report.html ../report          
      - name: Extract P(95)
        id: extract-p95
        shell: bash
        run: |
          set -e
          echo "p95=$(grep http_req_duration load-testing/test-output.log | awk -F 'p\\(95\\)=' '{split($2,a,\"ms\"); print a[1]}')" >> $GITHUB_OUTPUT
          echo $GITHUB_OUTPUT          
      - name: Debug failure
        if: failure()
        uses: ./.github/actions/kyverno-logs

  load-test:
    if: github.event_name == 'pull_request'
    needs:
      - prepare-images
      - old-load-test
    runs-on: ubuntu-latest
    permissions:
      packages: read
    strategy:
      fail-fast: false
      matrix:
        k8s-version: [v1.31.0]
    steps:
      - name: Checkout kyverno/kyverno
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Checkout kyverno/load-testing
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: kyverno/load-testing
          path: load-testing
      - name: Install Helm
        id: helm
        uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
      - name: Create Kind cluster
        uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
        with:
          node_image: kindest/node:${{ matrix.k8s-version }}
          cluster_name: kind
          config: ./scripts/config/kind/default.yaml
      - name: Download kyverno images archive
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          name: kyverno.tar
      - name: Load Kyverno images archive in Kind cluster
        shell: bash
        run: |
          set -e
          kind load image-archive kyverno.tar --name kind          
      - name: Install Kyverno
        shell: bash
        run: |
          set -e
          export HELM=${{ steps.helm.outputs.helm-path }}
          export USE_CONFIG=default-with-profiling
          export EXPLICIT_INSTALL_SETTINGS='--set admissionController.replicas=1 --set admissionController.resources.requests.cpu=100m --set admissionController.resources.limits.cpu=1500m --set admissionController.resources.requests.memory=128Mi --set admissionController.resources.limits.memory=384Mi'
          make kind-install-kyverno          
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Install K6
        shell: bash
        run: |
          set -e
          go install go.k6.io/xk6/cmd/xk6@latest
          $(go env GOPATH)/bin/xk6 build --with github.com/grafana/xk6-dashboard@latest
          mkdir -p $HOME/.local/bin && mv ./k6 $HOME/.local/bin
          echo "$HOME/.local/bin" >> $GITHUB_PATH          
      - name: Run load tests using K6
        shell: bash
        run: |
          set -e
          mkdir -p report
          KYVERNO_NODE_IP=$(kubectl get nodes -o jsonpath='{.items[?(@.metadata.labels.kubernetes\.io/hostname=="kind-control-plane")].status.addresses[?(@.type=="InternalIP")].address}')
          curl -s "http://$KYVERNO_NODE_IP:30950/debug/pprof/profile?seconds=90" > report/cpu.pprof &
          cd load-testing
          ./k6/run.sh k6/tests/kyverno-pss.js -e SCENARIO=average --out dashboard=export=load-report.html
          wait %1 || true
          mv load-report.html ../report          
      - name: Compare P(95)
        shell: bash
        run: |
          set -e
          echo "Old P(95): ${{ needs.old-load-test.outputs.p95 }}"
          OLD_NUM=${{ needs.old-load-test.outputs.p95 }}
          NEW_NUM=$(grep http_req_duration load-testing/test-output.log | awk -F 'p\\(95\\)=' '{split($2,a,"ms"); print a[1]}')
          echo "$OLD_NUM to $NEW_NUM"
          if [ $(echo "$OLD_NUM < $NEW_NUM" | bc) -eq 1 ]; then
            echo "P(95) increased from $OLD_NUM to $NEW_NUM"
            exit 1
          fi          
      - name: Archive Report
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: load-test-report.html
          path: report
      - name: Debug failure
        if: failure()
        uses: ./.github/actions/kyverno-logs

  scale-test:
    if: github.event_name == 'pull_request'
    needs:
      - define-matrix
      - prepare-images
    runs-on: ubuntu-latest
    permissions:
      packages: read
    strategy:
      fail-fast: false
      matrix:
        k8s-version: [v1.31.0]
        test: ${{ fromJson(needs.define-matrix.outputs.tests) }}
    steps:
      - name: Checkout kyverno/kyverno
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Checkout kyverno/load-testing
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: kyverno/load-testing
          path: load-testing
      - name: Install Helm
        id: helm
        uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
      - name: Create Kind cluster
        uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
        with:
          node_image: kindest/node:${{ matrix.k8s-version }}
          cluster_name: kind
          config: ./scripts/config/kind/default.yaml
      - name: Download kyverno images archive
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          name: kyverno.tar
      - name: Load Kyverno images archive in Kind cluster
        shell: bash
        run: |
          set -e
          kind load image-archive kyverno.tar --name kind          
      - name: Install Metrics Server and Prometheus
        shell: bash
        run: |
          set -e
          export HELM=${{ steps.helm.outputs.helm-path }}
          make dev-lab-metrics-server dev-lab-prometheus          
      - name: Install Kyverno
        shell: bash
        run: |
          set -e
          export HELM=${{ steps.helm.outputs.helm-path }}
          export USE_CONFIG=default-with-profiling
          export EXPLICIT_INSTALL_SETTINGS='--set admissionController.replicas=${{ matrix.test.replicas }} --set admissionController.serviceMonitor.enabled=true --set reportsController.serviceMonitor.enabled=true --set admissionController.container.resources.requests.cpu=${{ matrix.test.cpu_request }} --set admissionController.container.resources.requests.memory=${{ matrix.test.memory_request }} --set admissionController.container.resources.limits.memory=${{ matrix.test.memory_limit }} --set reportsController.resources.limits.memory=10Gi'
          make kind-install-kyverno          
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Install K6
        shell: bash
        run: |
          set -e
          go install go.k6.io/xk6/cmd/xk6@latest
          $(go env GOPATH)/bin/xk6 build --with github.com/grafana/xk6-dashboard@latest
          mkdir -p $HOME/.local/bin && mv ./k6 $HOME/.local/bin
          echo "$HOME/.local/bin" >> $GITHUB_PATH          
      - name: Run load tests using K6
        shell: bash
        run: |
          set -e
          mkdir -p report
          KYVERNO_NODE_IP=$(kubectl get nodes -o jsonpath='{.items[?(@.metadata.labels.kubernetes\.io/hostname=="kind-control-plane")].status.addresses[?(@.type=="InternalIP")].address}')
          curl -s "http://$KYVERNO_NODE_IP:30950/debug/pprof/profile?seconds=30" > report/cpu.pprof &
          cd load-testing
          ./k6/run.sh k6/tests/${{ matrix.test.name }}.js -e SCENARIO=${{ matrix.test.scenario }} --vus ${{ matrix.test.concurrent_connections }} --iterations ${{ matrix.test.total_iterations }} ${{ matrix.test.extra_options }} --out dashboard=export=load-report.html
          wait %1 || true
          mv load-report.html ../report          
      - name: Collect Resource Metrics
        shell: bash
        run: |
          set -e
          kubectl port-forward --address 127.0.0.1 svc/kube-prometheus-stack-prometheus 9090:9090 -n monitoring &
          sleep 3
          curl -s "http://127.0.0.1:9090/prometheus/api/v1/query?query=$(echo -n "rate(container_cpu_usage_seconds_total{image=\"$(make kind-admission-controller-image-name)\"}[1m])" | jq -sRr @uri)" > report/cpu-usage.json
          curl -s "http://127.0.0.1:9090/prometheus/api/v1/query?query=$(echo -n "max_over_time(container_memory_working_set_bytes{image=\"$(make kind-admission-controller-image-name)\"}[1m])/(2^20)" | jq -sRr @uri)" > report/memory-usage.json
          kill %1 || true          
      - name: Collect Report Metrics
        shell: bash
        run: |
          set -e
          sleep 60
          ./test/load/k6/reports-size-in-etcd.sh > report/reports-size-in-etcd.txt          
      - name: Archive Report
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: report-${{ matrix.k8s-version }}-${{ matrix.test.name }}-${{ matrix.test.scenario }}-${{ matrix.test.replicas }}-${{ matrix.test.cpu_request }}-${{ matrix.test.memory_request }}-${{ matrix.test.memory_limit }}-${{ matrix.test.concurrent_connections }}
          path: report
      - name: Debug failure
        # if: failure()
        uses: ./.github/actions/kyverno-logs