2022-10-20 14:17:33 +02:00
|
|
|
name: Conformance tests
|
2023-01-02 21:51:18 +01:00
|
|
|
|
2022-10-20 14:17:33 +02:00
|
|
|
on:
|
|
|
|
pull_request:
|
|
|
|
branches:
|
|
|
|
- 'main'
|
|
|
|
- 'release*'
|
|
|
|
|
2023-06-21 11:47:15 +02:00
|
|
|
permissions: {}
|
|
|
|
|
2022-11-23 18:59:57 +05:30
|
|
|
concurrency:
|
|
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
|
|
cancel-in-progress: true
|
|
|
|
|
2022-10-20 14:17:33 +02:00
|
|
|
jobs:
|
2023-02-28 10:33:53 +01:00
|
|
|
prepare-images:
|
|
|
|
runs-on: ubuntu-latest
|
2023-06-21 11:47:15 +02:00
|
|
|
permissions:
|
|
|
|
contents: read
|
2023-02-28 10:33:53 +01:00
|
|
|
steps:
|
|
|
|
- name: Checkout
|
2023-06-12 08:11:03 +00:00
|
|
|
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
2023-02-28 10:33:53 +01:00
|
|
|
- name: Setup build env
|
|
|
|
uses: ./.github/actions/setup-build-env
|
|
|
|
with:
|
|
|
|
build-cache-key: build-images
|
|
|
|
- name: ko build
|
2023-04-07 14:10:53 +02:00
|
|
|
run: VERSION=${{ github.ref_name }} make docker-save-image-all
|
2023-02-28 10:33:53 +01:00
|
|
|
- name: upload images archive
|
|
|
|
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
|
|
|
|
with:
|
|
|
|
name: kyverno.tar
|
|
|
|
path: kyverno.tar
|
|
|
|
retention-days: 1
|
|
|
|
if-no-files-found: error
|
|
|
|
|
2023-04-10 22:30:49 +02:00
|
|
|
# runs conformance test suites with configuration:
|
2023-04-13 11:08:54 +02:00
|
|
|
standard:
|
2023-06-21 11:47:15 +02:00
|
|
|
runs-on: ubuntu-latest
|
|
|
|
permissions:
|
|
|
|
contents: read
|
2022-12-21 18:14:53 +08:00
|
|
|
strategy:
|
|
|
|
fail-fast: false
|
|
|
|
matrix:
|
2023-04-13 11:08:54 +02:00
|
|
|
config:
|
|
|
|
- name: standard
|
|
|
|
values:
|
|
|
|
- standard
|
2023-02-28 15:33:46 +01:00
|
|
|
k8s-version:
|
2023-04-13 11:08:54 +02:00
|
|
|
- name: v1.24
|
2023-04-13 12:21:54 +02:00
|
|
|
version: v1.24.12
|
2023-04-13 11:08:54 +02:00
|
|
|
- name: v1.25
|
2023-04-13 12:21:54 +02:00
|
|
|
version: v1.25.8
|
2023-04-13 11:08:54 +02:00
|
|
|
- name: v1.26
|
2023-04-13 12:21:54 +02:00
|
|
|
version: v1.26.3
|
2023-04-13 11:08:54 +02:00
|
|
|
- name: v1.27
|
2023-04-24 13:55:17 +02:00
|
|
|
version: v1.27.1
|
2023-02-28 15:33:46 +01:00
|
|
|
tests:
|
|
|
|
- autogen
|
2023-06-29 14:53:23 +02:00
|
|
|
- background-only
|
2023-02-28 15:33:46 +01:00
|
|
|
- cleanup
|
2023-06-27 09:58:50 -07:00
|
|
|
- deferred
|
2023-02-28 15:33:46 +01:00
|
|
|
- events
|
|
|
|
- exceptions
|
2023-04-13 17:50:35 +08:00
|
|
|
- generate/clusterpolicy
|
|
|
|
- generate/policy
|
|
|
|
- generate/validation
|
2023-02-28 15:33:46 +01:00
|
|
|
- mutate
|
2023-03-22 18:12:10 +01:00
|
|
|
- policy-validation
|
2023-02-28 15:33:46 +01:00
|
|
|
- rangeoperators
|
|
|
|
- rbac
|
|
|
|
- reports
|
|
|
|
- validate
|
2023-03-27 18:19:19 +02:00
|
|
|
- verify-manifests
|
2023-02-28 15:33:46 +01:00
|
|
|
- verifyImages
|
|
|
|
- webhooks
|
2023-02-28 10:33:53 +01:00
|
|
|
needs: prepare-images
|
2023-04-13 11:08:54 +02:00
|
|
|
name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
|
2022-10-20 14:17:33 +02:00
|
|
|
steps:
|
|
|
|
- name: Checkout
|
2023-06-12 08:11:03 +00:00
|
|
|
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
2023-01-05 16:52:52 +01:00
|
|
|
- name: Setup build env
|
|
|
|
uses: ./.github/actions/setup-build-env
|
2022-10-20 14:17:33 +02:00
|
|
|
with:
|
2023-01-05 16:52:52 +01:00
|
|
|
build-cache-key: run-conformance
|
2023-02-28 10:33:53 +01:00
|
|
|
- name: Create kind cluster
|
|
|
|
run: |
|
2023-04-13 11:08:54 +02:00
|
|
|
export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
|
2023-02-28 10:33:53 +01:00
|
|
|
make kind-create-cluster
|
|
|
|
- name: Download kyverno images archive
|
|
|
|
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
|
2023-01-05 23:36:13 +01:00
|
|
|
with:
|
2023-02-28 10:33:53 +01:00
|
|
|
name: kyverno.tar
|
|
|
|
- name: Load kyverno images archive in kind cluster
|
|
|
|
run: make kind-load-image-archive
|
|
|
|
- name: Install kyverno
|
2023-04-13 11:08:54 +02:00
|
|
|
run: |
|
|
|
|
export USE_CONFIG=${{ join(matrix.config.values, ',') }}
|
|
|
|
make kind-install-kyverno
|
2023-02-28 10:33:53 +01:00
|
|
|
- name: Wait for kyverno ready
|
|
|
|
uses: ./.github/actions/kyverno-wait-ready
|
2022-11-04 10:00:31 -04:00
|
|
|
- name: Test with kuttl
|
2023-02-28 15:33:46 +01:00
|
|
|
run: |
|
|
|
|
./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }} \
|
|
|
|
--config ./test/conformance/kuttl/_config/common.yaml
|
2022-11-14 21:34:56 +01:00
|
|
|
- name: Debug failure
|
|
|
|
if: failure()
|
2023-01-05 12:35:04 +01:00
|
|
|
uses: ./.github/actions/kyverno-logs
|
2023-04-10 22:30:49 +02:00
|
|
|
|
|
|
|
# runs conformance test suites with configuration:
|
|
|
|
force-failure-policy-ignore:
|
2023-06-21 11:47:15 +02:00
|
|
|
runs-on: ubuntu-latest
|
|
|
|
permissions:
|
|
|
|
contents: read
|
2023-04-10 22:30:49 +02:00
|
|
|
strategy:
|
|
|
|
fail-fast: false
|
|
|
|
matrix:
|
2023-04-13 11:08:54 +02:00
|
|
|
config:
|
|
|
|
- name: force-failure-policy-ignore
|
|
|
|
values:
|
|
|
|
- standard
|
|
|
|
- force-failure-policy-ignore
|
2023-04-10 22:30:49 +02:00
|
|
|
k8s-version:
|
2023-04-13 11:08:54 +02:00
|
|
|
- name: v1.24
|
2023-04-13 12:21:54 +02:00
|
|
|
version: v1.24.12
|
2023-04-13 11:08:54 +02:00
|
|
|
- name: v1.25
|
2023-04-13 12:21:54 +02:00
|
|
|
version: v1.25.8
|
2023-04-13 11:08:54 +02:00
|
|
|
- name: v1.26
|
2023-04-13 12:21:54 +02:00
|
|
|
version: v1.26.3
|
2023-04-13 11:08:54 +02:00
|
|
|
- name: v1.27
|
2023-04-24 13:55:17 +02:00
|
|
|
version: v1.27.1
|
2023-04-10 22:30:49 +02:00
|
|
|
tests:
|
|
|
|
- force-failure-policy-ignore
|
|
|
|
- rbac
|
|
|
|
needs: prepare-images
|
2023-04-13 11:08:54 +02:00
|
|
|
name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
|
2023-04-10 22:30:49 +02:00
|
|
|
steps:
|
|
|
|
- name: Checkout
|
2023-06-12 08:11:03 +00:00
|
|
|
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
2023-04-10 22:30:49 +02:00
|
|
|
- name: Setup build env
|
|
|
|
uses: ./.github/actions/setup-build-env
|
|
|
|
with:
|
|
|
|
build-cache-key: run-conformance
|
|
|
|
- name: Create kind cluster
|
|
|
|
run: |
|
2023-04-13 11:08:54 +02:00
|
|
|
export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
|
2023-04-10 22:30:49 +02:00
|
|
|
make kind-create-cluster
|
|
|
|
- name: Download kyverno images archive
|
|
|
|
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
|
|
|
|
with:
|
|
|
|
name: kyverno.tar
|
|
|
|
- name: Load kyverno images archive in kind cluster
|
|
|
|
run: make kind-load-image-archive
|
|
|
|
- name: Install kyverno
|
|
|
|
run: |
|
2023-04-13 11:08:54 +02:00
|
|
|
export USE_CONFIG=${{ join(matrix.config.values, ',') }}
|
2023-04-10 22:30:49 +02:00
|
|
|
make kind-install-kyverno
|
|
|
|
- name: Wait for kyverno ready
|
|
|
|
uses: ./.github/actions/kyverno-wait-ready
|
|
|
|
- name: Test with kuttl
|
|
|
|
run: |
|
|
|
|
./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }} \
|
|
|
|
--config ./test/conformance/kuttl/_config/common.yaml
|
|
|
|
- name: Debug failure
|
|
|
|
if: failure()
|
|
|
|
uses: ./.github/actions/kyverno-logs
|
2023-04-11 14:50:53 +02:00
|
|
|
|
|
|
|
# runs conformance test suites with configuration:
|
|
|
|
default:
|
2023-06-21 11:47:15 +02:00
|
|
|
runs-on: ubuntu-latest
|
|
|
|
permissions:
|
|
|
|
contents: read
|
2023-04-11 14:50:53 +02:00
|
|
|
strategy:
|
|
|
|
fail-fast: false
|
|
|
|
matrix:
|
2023-04-13 11:08:54 +02:00
|
|
|
config:
|
|
|
|
- name: default
|
|
|
|
values:
|
|
|
|
- default
|
2023-04-11 14:50:53 +02:00
|
|
|
k8s-version:
|
2023-04-13 11:08:54 +02:00
|
|
|
- name: v1.24
|
2023-04-13 12:21:54 +02:00
|
|
|
version: v1.24.12
|
2023-04-13 11:08:54 +02:00
|
|
|
- name: v1.25
|
2023-04-13 12:21:54 +02:00
|
|
|
version: v1.25.8
|
2023-04-13 11:08:54 +02:00
|
|
|
- name: v1.26
|
2023-04-13 12:21:54 +02:00
|
|
|
version: v1.26.3
|
2023-04-13 11:08:54 +02:00
|
|
|
- name: v1.27
|
2023-04-24 13:55:17 +02:00
|
|
|
version: v1.27.1
|
2023-04-11 14:50:53 +02:00
|
|
|
tests:
|
|
|
|
- rbac
|
|
|
|
needs: prepare-images
|
2023-04-13 11:08:54 +02:00
|
|
|
name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
|
2023-04-11 14:50:53 +02:00
|
|
|
steps:
|
|
|
|
- name: Checkout
|
2023-06-12 08:11:03 +00:00
|
|
|
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
2023-04-11 14:50:53 +02:00
|
|
|
- name: Setup build env
|
|
|
|
uses: ./.github/actions/setup-build-env
|
|
|
|
with:
|
|
|
|
build-cache-key: run-conformance
|
|
|
|
- name: Create kind cluster
|
|
|
|
run: |
|
2023-04-13 11:08:54 +02:00
|
|
|
export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
|
2023-04-11 14:50:53 +02:00
|
|
|
make kind-create-cluster
|
|
|
|
- name: Download kyverno images archive
|
|
|
|
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
|
|
|
|
with:
|
|
|
|
name: kyverno.tar
|
|
|
|
- name: Load kyverno images archive in kind cluster
|
|
|
|
run: make kind-load-image-archive
|
|
|
|
- name: Install kyverno
|
|
|
|
run: |
|
2023-04-13 11:08:54 +02:00
|
|
|
export USE_CONFIG=${{ join(matrix.config.values, ',') }}
|
2023-04-11 14:50:53 +02:00
|
|
|
make kind-install-kyverno
|
|
|
|
- name: Wait for kyverno ready
|
|
|
|
uses: ./.github/actions/kyverno-wait-ready
|
|
|
|
- name: Test with kuttl
|
|
|
|
run: |
|
|
|
|
./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }} \
|
|
|
|
--config ./test/conformance/kuttl/_config/common.yaml
|
|
|
|
- name: Debug failure
|
|
|
|
if: failure()
|
|
|
|
uses: ./.github/actions/kyverno-logs
|
2023-06-19 11:17:24 -04:00
|
|
|
|
|
|
|
# runs conformance test suites with configuration:
|
|
|
|
policy-library:
|
2023-06-21 11:47:15 +02:00
|
|
|
runs-on: ubuntu-latest
|
|
|
|
permissions:
|
|
|
|
contents: read
|
2023-06-19 11:17:24 -04:00
|
|
|
strategy:
|
|
|
|
fail-fast: false
|
|
|
|
matrix:
|
|
|
|
config:
|
|
|
|
- name: policy-library
|
|
|
|
values:
|
|
|
|
- standard
|
|
|
|
k8s-version:
|
|
|
|
- name: v1.24
|
|
|
|
version: v1.24.12
|
|
|
|
- name: v1.25
|
|
|
|
version: v1.25.8
|
|
|
|
- name: v1.26
|
|
|
|
version: v1.26.3
|
|
|
|
- name: v1.27
|
|
|
|
version: v1.27.1
|
|
|
|
needs: prepare-images
|
|
|
|
name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }}
|
|
|
|
steps:
|
|
|
|
- name: Checkout kyverno/kyverno
|
|
|
|
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
|
|
|
- name: Checkout kyverno/policies
|
|
|
|
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
|
|
|
|
with:
|
|
|
|
repository: kyverno/policies
|
|
|
|
path: policies
|
|
|
|
- name: Setup build env
|
|
|
|
uses: ./.github/actions/setup-build-env
|
|
|
|
with:
|
|
|
|
build-cache-key: run-conformance
|
|
|
|
- name: Create kind cluster
|
|
|
|
run: |
|
|
|
|
export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
|
|
|
|
make kind-create-cluster
|
|
|
|
- name: Download kyverno images archive
|
|
|
|
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
|
|
|
|
with:
|
|
|
|
name: kyverno.tar
|
|
|
|
- name: Load kyverno images archive in kind cluster
|
|
|
|
run: make kind-load-image-archive
|
|
|
|
- name: Install kyverno
|
|
|
|
run: |
|
|
|
|
export USE_CONFIG=${{ join(matrix.config.values, ',') }}
|
|
|
|
make kind-install-kyverno
|
|
|
|
- name: Wait for kyverno ready
|
|
|
|
uses: ./.github/actions/kyverno-wait-ready
|
|
|
|
- name: Test policy library with kuttl
|
|
|
|
run: |
|
|
|
|
cd policies
|
|
|
|
../.tools/kubectl-kuttl test . --config ./kuttl-test.yaml
|
|
|
|
- name: Debug failure
|
|
|
|
if: failure()
|
|
|
|
uses: ./.github/actions/kyverno-logs
|