From 38feb7d6948e45e5e5bfc24e80641d24f53db321 Mon Sep 17 00:00:00 2001 From: Khaled Emara Date: Mon, 29 Jan 2024 14:30:51 +0200 Subject: [PATCH] ci: add load testing workflow (#9515) Signed-off-by: Khaled Emara --- .github/workflows/load-testing.yml | 141 ++++++++++++++++++ Makefile | 9 ++ .../default-with-profiling/kyverno.yaml | 5 + scripts/config/kind/default.yaml | 3 + .../standard-with-profiling/kyverno.yaml | 47 ++++++ 5 files changed, 205 insertions(+) create mode 100644 .github/workflows/load-testing.yml create mode 100644 scripts/config/default-with-profiling/kyverno.yaml create mode 100644 scripts/config/standard-with-profiling/kyverno.yaml diff --git a/.github/workflows/load-testing.yml b/.github/workflows/load-testing.yml new file mode 100644 index 0000000000..f9b223fa65 --- /dev/null +++ b/.github/workflows/load-testing.yml @@ -0,0 +1,141 @@ +name: Baseline Load Tests + +permissions: {} + +on: + pull_request: + branches: + - "main" + - "release*" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + prepare-images: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Setup caches + uses: ./.github/actions/setup-caches + timeout-minutes: 5 + continue-on-error: true + with: + build-cache-key: build-images + - name: Setup build env + uses: ./.github/actions/setup-build-env + timeout-minutes: 10 + with: + free-disk-space: false + - name: ko build + shell: bash + run: | + set -e + VERSION=${{ github.ref_name }} make docker-save-image-all + - name: upload images archive + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: kyverno.tar + path: kyverno.tar + retention-days: 1 + if-no-files-found: error + + load-test: + runs-on: ubuntu-latest + permissions: + packages: read + strategy: + fail-fast: false + matrix: + k8s-version: + - name: v1.29 + version: v1.29.0 + kyverno-config: + - name: default + values: + - default-with-profiling + - name: standard + values: + - standard-with-profiling + test: + - kyverno-pss + - kyverno-mutate + k6-config: + - vus: 5 + iterations: 100 + - vus: 10 + iterations: 200 + needs: + - prepare-images + name: ${{ matrix.kyverno-config.name }} - ${{ matrix.test }} - ${{ matrix.k6-config.vus }} vus - ${{ matrix.k6-config.iterations }} iterations + steps: + - name: Checkout kyverno/kyverno + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Checkout kyverno/load-testing + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: kyverno/load-testing + path: load-testing + - name: Install helm + id: helm + uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Create kind cluster + uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 + with: + node_image: kindest/node:${{ matrix.k8s-version.version }} + cluster_name: kind + config: ./scripts/config/kind/default.yaml + - name: Download kyverno images archive + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: kyverno.tar + - name: Load kyverno images archive in kind cluster + shell: bash + run: | + set -e + kind load image-archive kyverno.tar --name kind + - name: Install kyverno + shell: bash + run: | + set -e + export HELM=${{ steps.helm.outputs.helm-path }} + export USE_CONFIG=${{ join(matrix.kyverno-config.values, ',') }} + make kind-install-kyverno + bash load-testing/k8s/metrics-server/hack.sh + make kind-install-goldilocks + - name: Wait for kyverno ready + uses: ./.github/actions/kyverno-wait-ready + - name: Run load tests using K6 + shell: bash + run: | + set -e + KYVERNO_NODE_IP=$(kubectl get nodes -o jsonpath='{.items[?(@.metadata.labels.kubernetes\.io/hostname=="kind-control-plane")].status.addresses[?(@.type=="InternalIP")].address}') + curl http://$KYVERNO_NODE_IP:30950/debug/pprof/heap > heap.pprof + curl "http://$KYVERNO_NODE_IP:30950/debug/pprof/profile?seconds=30" > cpu.pprof 2> curl.tmp & + cd load-testing/k6 + ./start.sh tests/${{ matrix.test }}.js ${{ matrix.k6-config.vus }} ${{ matrix.k6-config.iterations }} + wait %1 || true + # TODO: wait for VPA to stabilize and recommend + kubectl -n kyverno get vpa goldilocks-kyverno-admission-controller -o jsonpath='{.status.recommendation.containerRecommendations[*]}' + - name: Archive load test results + if: failure() + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: load-test-reports + path: load-testing/k6/${{ matrix.test }}.js-${{ matrix.k6-config.vus }}vu-${{ matrix.k6-config.iterations }}it-logs.txt + - name: Archive pprof CPU profiles + if: failure() + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: pprof-cpu-profiles + path: cpu.pprof + - name: Archive pprof HEAP profiles + if: failure() + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: pprof-heap-profiles + path: heap.pprof diff --git a/Makefile b/Makefile index 5e6d1c8095..35ff79e8b0 100644 --- a/Makefile +++ b/Makefile @@ -960,6 +960,15 @@ kind-install-kyverno: $(HELM) ## Install kyverno helm chart --set crds.migration.image.tag=$(GIT_SHA) \ $(foreach CONFIG,$(subst $(COMMA), ,$(USE_CONFIG)),--values ./scripts/config/$(CONFIG)/kyverno.yaml) +.PHONY: kind-install-goldilocks +kind-install-goldilocks: $(HELM) ## Install goldilocks helm chart + @echo Install goldilocks chart... >&2 + @$(HELM) upgrade --install vpa --namespace vpa --create-namespace --wait \ + --repo https://charts.fairwinds.com/stable vpa + @$(HELM) upgrade --install goldilocks --namespace goldilocks --create-namespace --wait \ + --repo https://charts.fairwinds.com/stable goldilocks + kubectl label ns kyverno goldilocks.fairwinds.com/enabled=true + .PHONY: kind-deploy-kyverno kind-deploy-kyverno: $(HELM) kind-load-all ## Build images, load them in kind cluster and deploy kyverno helm chart @$(MAKE) kind-install-kyverno diff --git a/scripts/config/default-with-profiling/kyverno.yaml b/scripts/config/default-with-profiling/kyverno.yaml new file mode 100644 index 0000000000..37738e2abc --- /dev/null +++ b/scripts/config/default-with-profiling/kyverno.yaml @@ -0,0 +1,5 @@ +admissionController: + profiling: + enabled: true + serviceType: NodePort + nodePort: 30950 diff --git a/scripts/config/kind/default.yaml b/scripts/config/kind/default.yaml index 9438061e5d..1e1322d51e 100644 --- a/scripts/config/kind/default.yaml +++ b/scripts/config/kind/default.yaml @@ -34,3 +34,6 @@ nodes: - role: worker - role: worker - role: worker +featureGates: + "JobPodFailurePolicy": true + "PodDisruptionConditions": true diff --git a/scripts/config/standard-with-profiling/kyverno.yaml b/scripts/config/standard-with-profiling/kyverno.yaml new file mode 100644 index 0000000000..7f21fad15f --- /dev/null +++ b/scripts/config/standard-with-profiling/kyverno.yaml @@ -0,0 +1,47 @@ +features: + policyExceptions: + enabled: true + +admissionController: + profiling: + enabled: true + serviceType: NodePort + nodePort: 30950 + +backgroundController: + rbac: + clusterRole: + extraResources: + - apiGroups: + - "*" + resources: + - configmaps + - networkpolicies + - resourcequotas + - secrets + - roles + - rolebindings + - limitranges + - namespaces + - nodes + - nodes/status + - pods + verbs: + - create + - update + - patch + - delete + - get + - list + +cleanupController: + rbac: + clusterRole: + extraResources: + - apiGroups: + - "" + resources: + - pods + verbs: + - list + - delete