From 61c4a3e7b74b0c076aced439ec09c63d278d4d29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?=
<charles.edouard@nirmata.com>
Date: Mon, 2 Sep 2024 19:55:55 +0200
Subject: [PATCH] chore: split conformance jobs per folder (#10974)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* chore: simplify e2e matrix
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* chore: use dynamic matrix
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* chore: split conformance jobs per folder
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
---------
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
---
.github/actions/run-tests/action.yaml | 62 ++++
.github/workflows/conformance.yaml | 486 ++++++++++++++++++++++----
2 files changed, 472 insertions(+), 76 deletions(-)
create mode 100644 .github/actions/run-tests/action.yaml
diff --git a/.github/actions/run-tests/action.yaml b/.github/actions/run-tests/action.yaml
new file mode 100644
index 0000000000..c0b927be10
--- /dev/null
+++ b/.github/actions/run-tests/action.yaml
@@ -0,0 +1,62 @@
+name: Runs E2E Tests
+description: Runs E2E tests using chainsaw
+inputs:
+ k8s-version:
+ description: Kubernetes version
+ required: true
+ config:
+ description: Chainsaw config
+ required: true
+ tests:
+ description: Test regex
+ required: true
+ token:
+ description: GH token
+ required: true
+runs:
+ using: composite
+ steps:
+ # install tools
+ - name: Install helm
+ id: helm
+ uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
+ - name: Install chainsaw
+ uses: kyverno/action-install-chainsaw@82d8e747037f840e0ef9bdd97ecdc617f5535bdc # v0.2.8
+ # create cluster
+ - name: Create kind cluster
+ uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+ with:
+ node_image: kindest/node:${{ inputs.k8s-version }}
+ cluster_name: kind
+ config: ./scripts/config/kind/default.yaml
+ # deploy kyverno
+ - name: Download kyverno images archive
+ uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+ with:
+ name: kyverno.tar
+ - name: Load kyverno images archive in kind cluster
+ shell: bash
+ run: |
+ set -e
+ kind load image-archive kyverno.tar --name kind
+ - name: Install kyverno
+ shell: bash
+ run: |
+ set -e
+ export HELM=${{ steps.helm.outputs.helm-path }}
+ export USE_CONFIG=standard
+ make kind-install-kyverno
+ - name: Wait for kyverno ready
+ uses: ./.github/actions/kyverno-wait-ready
+ # run tests
+ - name: Test with Chainsaw
+ shell: bash
+ env:
+ GITHUB_TOKEN: ${{ inputs.token }}
+ run: |
+ set -e
+ cd ./test/conformance/chainsaw && chainsaw test --include-test-regex '^chainsaw$/${{ inputs.tests }}' --config ../../../.chainsaw.yaml
+ # debug
+ - name: Debug failure
+ if: failure()
+ uses: ./.github/actions/kyverno-logs
\ No newline at end of file
diff --git a/.github/workflows/conformance.yaml b/.github/workflows/conformance.yaml
index 8c53e00c12..a22b8987dd 100644
--- a/.github/workflows/conformance.yaml
+++ b/.github/workflows/conformance.yaml
@@ -84,91 +84,385 @@ jobs:
retention-days: 1
if-no-files-found: error
- standard:
+ assert:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
- k8s-version:
- # - v1.28.13
- # - v1.29.8
- # - v1.30.4
- - v1.31.0
- tests:
- - ${{ fromJSON(needs.define-matrix.outputs.tests).assert }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).autogen }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).background-only }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).cleanup }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).deferred }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).events }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).exceptions }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).filter }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).generate }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).globalcontext }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).lease }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).mutate }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).policy-validation }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).rangeoperators }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).rbac }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).reports }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).validate }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).verify-manifests }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).verifyImages }}
- - ${{ fromJSON(needs.define-matrix.outputs.tests).webhooks }}
- needs:
- - prepare-images
- - define-matrix
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).assert }}
+ needs: [ prepare-images, define-matrix ]
steps:
- - name: Checkout
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- # install tools
- - name: Install helm
- id: helm
- uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
- - name: Install chainsaw
- uses: kyverno/action-install-chainsaw@82d8e747037f840e0ef9bdd97ecdc617f5535bdc # v0.2.8
- # create cluster
- - name: Create kind cluster
- uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+
+ autogen:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).autogen }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
with:
- node_image: kindest/node:${{ matrix.k8s-version }}
- cluster_name: kind
- config: ./scripts/config/kind/default.yaml
- # deploy kyverno
- - name: Download kyverno images archive
- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ background-only:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).background-only }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
with:
- name: kyverno.tar
- - name: Load kyverno images archive in kind cluster
- shell: bash
- run: |
- set -e
- kind load image-archive kyverno.tar --name kind
- - name: Install kyverno
- shell: bash
- run: |
- set -e
- export HELM=${{ steps.helm.outputs.helm-path }}
- export USE_CONFIG=standard
- make kind-install-kyverno
- - name: Wait for kyverno ready
- uses: ./.github/actions/kyverno-wait-ready
- # run tests
- - name: Test with Chainsaw
- shell: bash
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- set -e
- cd ./test/conformance/chainsaw && chainsaw test --include-test-regex '^chainsaw$/${{ matrix.tests }}' --config ../../../.chainsaw.yaml
- # debug
- - name: Debug failure
- if: failure()
- uses: ./.github/actions/kyverno-logs
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ cleanup:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).cleanup }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ deferred:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).deferred }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ events:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).events }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ exceptions:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).exceptions }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ filter:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).filter }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ generate:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).generate }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ globalcontext:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).globalcontext }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ lease:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).lease }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ mutate:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).mutate }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ policy-validation:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).policy-validation }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ rangeoperators:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).rangeoperators }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ rbac:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).rbac }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ reports:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).reports }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ validate:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).validate }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ verify-manifests:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).verify-manifests }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ verifyImages:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).verifyImages }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ webhooks:
+ runs-on: ubuntu-latest
+ permissions:
+ packages: read
+ strategy:
+ fail-fast: false
+ matrix:
+ k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
+ tests: ${{ fromJSON(needs.define-matrix.outputs.tests).webhooks }}
+ needs: [ prepare-images, define-matrix ]
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/run-tests
+ with:
+ k8s-version: ${{ matrix.k8s-version }}
+ config: standard
+ tests: ${{ matrix.tests }}
+ token: ${{ secrets.GITHUB_TOKEN }}
k8s-version-specific-tests-above-1-28:
runs-on: ubuntu-latest
@@ -797,7 +1091,27 @@ jobs:
conformance-required-success:
name: conformance-required
needs:
- - standard
+ - assert
+ - autogen
+ - background-only
+ - cleanup
+ - deferred
+ - events
+ - exceptions
+ - filter
+ - generate
+ - globalcontext
+ - lease
+ - mutate
+ - policy-validation
+ - rangeoperators
+ - rbac
+ - reports
+ - ttl
+ - validate
+ - verify-manifests
+ - verifyImages
+ - webhooks
- ttl
- force-failure-policy-ignore
- k8s-version-specific-tests-above-1-28
@@ -814,7 +1128,27 @@ jobs:
conformance-required-failure:
name: conformance-required
needs:
- - standard
+ - assert
+ - autogen
+ - background-only
+ - cleanup
+ - deferred
+ - events
+ - exceptions
+ - filter
+ - generate
+ - globalcontext
+ - lease
+ - mutate
+ - policy-validation
+ - rangeoperators
+ - rbac
+ - reports
+ - ttl
+ - validate
+ - verify-manifests
+ - verifyImages
+ - webhooks
- ttl
- force-failure-policy-ignore
- k8s-version-specific-tests-above-1-28