1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-03-29 02:45:06 +00:00

chore: split conformance jobs per folder (#10974)

* chore: simplify e2e matrix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* chore: use dynamic matrix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* chore: split conformance jobs per folder

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
This commit is contained in:
Charles-Edouard Brétéché 2024-09-02 19:55:55 +02:00 committed by GitHub
parent e4afa3a125
commit 61c4a3e7b7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 472 additions and 76 deletions

62
.github/actions/run-tests/action.yaml vendored Normal file
View file

@ -0,0 +1,62 @@
name: Runs E2E Tests
description: Runs E2E tests using chainsaw
inputs:
k8s-version:
description: Kubernetes version
required: true
config:
description: Chainsaw config
required: true
tests:
description: Test regex
required: true
token:
description: GH token
required: true
runs:
using: composite
steps:
# install tools
- name: Install helm
id: helm
uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
- name: Install chainsaw
uses: kyverno/action-install-chainsaw@82d8e747037f840e0ef9bdd97ecdc617f5535bdc # v0.2.8
# create cluster
- name: Create kind cluster
uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
with:
node_image: kindest/node:${{ inputs.k8s-version }}
cluster_name: kind
config: ./scripts/config/kind/default.yaml
# deploy kyverno
- name: Download kyverno images archive
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: kyverno.tar
- name: Load kyverno images archive in kind cluster
shell: bash
run: |
set -e
kind load image-archive kyverno.tar --name kind
- name: Install kyverno
shell: bash
run: |
set -e
export HELM=${{ steps.helm.outputs.helm-path }}
export USE_CONFIG=standard
make kind-install-kyverno
- name: Wait for kyverno ready
uses: ./.github/actions/kyverno-wait-ready
# run tests
- name: Test with Chainsaw
shell: bash
env:
GITHUB_TOKEN: ${{ inputs.token }}
run: |
set -e
cd ./test/conformance/chainsaw && chainsaw test --include-test-regex '^chainsaw$/${{ inputs.tests }}' --config ../../../.chainsaw.yaml
# debug
- name: Debug failure
if: failure()
uses: ./.github/actions/kyverno-logs

View file

@ -84,91 +84,385 @@ jobs:
retention-days: 1
if-no-files-found: error
standard:
assert:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version:
# - v1.28.13
# - v1.29.8
# - v1.30.4
- v1.31.0
tests:
- ${{ fromJSON(needs.define-matrix.outputs.tests).assert }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).autogen }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).background-only }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).cleanup }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).deferred }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).events }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).exceptions }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).filter }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).generate }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).globalcontext }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).lease }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).mutate }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).policy-validation }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).rangeoperators }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).rbac }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).reports }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).validate }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).verify-manifests }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).verifyImages }}
- ${{ fromJSON(needs.define-matrix.outputs.tests).webhooks }}
needs:
- prepare-images
- define-matrix
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).assert }}
needs: [ prepare-images, define-matrix ]
steps:
- name: Checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
# install tools
- name: Install helm
id: helm
uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
- name: Install chainsaw
uses: kyverno/action-install-chainsaw@82d8e747037f840e0ef9bdd97ecdc617f5535bdc # v0.2.8
# create cluster
- name: Create kind cluster
uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
autogen:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).autogen }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
node_image: kindest/node:${{ matrix.k8s-version }}
cluster_name: kind
config: ./scripts/config/kind/default.yaml
# deploy kyverno
- name: Download kyverno images archive
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
background-only:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).background-only }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
name: kyverno.tar
- name: Load kyverno images archive in kind cluster
shell: bash
run: |
set -e
kind load image-archive kyverno.tar --name kind
- name: Install kyverno
shell: bash
run: |
set -e
export HELM=${{ steps.helm.outputs.helm-path }}
export USE_CONFIG=standard
make kind-install-kyverno
- name: Wait for kyverno ready
uses: ./.github/actions/kyverno-wait-ready
# run tests
- name: Test with Chainsaw
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -e
cd ./test/conformance/chainsaw && chainsaw test --include-test-regex '^chainsaw$/${{ matrix.tests }}' --config ../../../.chainsaw.yaml
# debug
- name: Debug failure
if: failure()
uses: ./.github/actions/kyverno-logs
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
cleanup:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).cleanup }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
deferred:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).deferred }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
events:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).events }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
exceptions:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).exceptions }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
filter:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).filter }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
generate:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).generate }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
globalcontext:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).globalcontext }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
lease:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).lease }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
mutate:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).mutate }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
policy-validation:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).policy-validation }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
rangeoperators:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).rangeoperators }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
rbac:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).rbac }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
reports:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).reports }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
validate:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).validate }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
verify-manifests:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).verify-manifests }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
verifyImages:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).verifyImages }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
webhooks:
runs-on: ubuntu-latest
permissions:
packages: read
strategy:
fail-fast: false
matrix:
k8s-version: [ v1.28.13, v1.29.8, v1.30.4, v1.31.0 ]
tests: ${{ fromJSON(needs.define-matrix.outputs.tests).webhooks }}
needs: [ prepare-images, define-matrix ]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: ./.github/actions/run-tests
with:
k8s-version: ${{ matrix.k8s-version }}
config: standard
tests: ${{ matrix.tests }}
token: ${{ secrets.GITHUB_TOKEN }}
k8s-version-specific-tests-above-1-28:
runs-on: ubuntu-latest
@ -797,7 +1091,27 @@ jobs:
conformance-required-success:
name: conformance-required
needs:
- standard
- assert
- autogen
- background-only
- cleanup
- deferred
- events
- exceptions
- filter
- generate
- globalcontext
- lease
- mutate
- policy-validation
- rangeoperators
- rbac
- reports
- ttl
- validate
- verify-manifests
- verifyImages
- webhooks
- ttl
- force-failure-policy-ignore
- k8s-version-specific-tests-above-1-28
@ -814,7 +1128,27 @@ jobs:
conformance-required-failure:
name: conformance-required
needs:
- standard
- assert
- autogen
- background-only
- cleanup
- deferred
- events
- exceptions
- filter
- generate
- globalcontext
- lease
- mutate
- policy-validation
- rangeoperators
- rbac
- reports
- ttl
- validate
- verify-manifests
- verifyImages
- webhooks
- ttl
- force-failure-policy-ignore
- k8s-version-specific-tests-above-1-28