From fd45cc5eee85251e1d8830592b553250c8022cab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Thu, 7 Dec 2023 13:00:03 +0100 Subject: [PATCH] chore: improve cluster startup in conformance tests (#9103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: improve cluster startup in conformance tests Signed-off-by: Charles-Edouard Brétéché * fix flake Signed-off-by: Charles-Edouard Brétéché * fix flake Signed-off-by: Charles-Edouard Brétéché --------- Signed-off-by: Charles-Edouard Brétéché --- .github/workflows/conformance.yaml | 380 ++++++++---------- Makefile | 54 +-- .../{_config/common.yaml => .chainsaw.yaml} | 0 .../chainsaw-test.yaml | 2 + .../chainsaw-test.yaml | 2 + .../standard/keyed-basic/chainsaw-test.yaml | 2 + .../standard/keyed-secret/chainsaw-test.yaml | 2 + .../chainsaw-test.yaml | 2 + .../chainsaw-test.yaml | 2 + .../chainsaw-test.yaml | 2 + .../chainsaw-test.yaml | 2 + .../chainsaw-test.yaml | 8 + 12 files changed, 223 insertions(+), 235 deletions(-) rename test/conformance/chainsaw/{_config/common.yaml => .chainsaw.yaml} (100%) diff --git a/.github/workflows/conformance.yaml b/.github/workflows/conformance.yaml index 3cb22874ec..4e86fcb022 100644 --- a/.github/workflows/conformance.yaml +++ b/.github/workflows/conformance.yaml @@ -116,27 +116,24 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Setup caches - uses: ./.github/actions/setup-caches - timeout-minutes: 5 - continue-on-error: true + # install tools + - name: Install helm + id: helm + uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 with: - build-cache-key: run-conformance - - name: Install kind - shell: bash - run: | - # For AMD64 / x86_64 - [ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - # For ARM64 - [ $(uname -m) = aarch64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-arm64 - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind + token: ${{ secrets.GITHUB_TOKEN }} + - name: Install chainsaw + uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 + with: + release: v0.0.8 + # create cluster - name: Create kind cluster - shell: bash - run: | - set -e - export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }} - make kind-create-cluster + uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 + with: + node_image: kindest/node:${{ matrix.k8s-version.version }} + cluster_name: kind + config: ./scripts/config/kind/default.yaml + # deploy kyverno - name: Download kyverno images archive uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: @@ -145,30 +142,29 @@ jobs: shell: bash run: | set -e - make kind-load-image-archive + kind load image-archive kyverno.tar --name kind - name: Install kyverno shell: bash run: | set -e + export HELM=${{ steps.helm.outputs.helm-path }} export USE_CONFIG=${{ join(matrix.config.values, ',') }} make kind-install-kyverno - name: Wait for kyverno ready uses: ./.github/actions/kyverno-wait-ready - - name: Install Chainsaw - uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 - with: - release: v0.0.8 + # run tests - name: Test with Chainsaw shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e - chainsaw test --config ./test/conformance/chainsaw/_config/common.yaml --test-dir ./test/conformance/chainsaw/${{ matrix.tests }} --no-color=false + cd ./test/conformance/chainsaw && chainsaw test --test-dir ./${{ matrix.tests }} + # debug - name: Debug failure if: failure() uses: ./.github/actions/kyverno-logs - + validatingadmissionpolicies-v1alpha1: runs-on: ubuntu-latest permissions: @@ -191,28 +187,24 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Setup caches - uses: ./.github/actions/setup-caches - timeout-minutes: 5 - continue-on-error: true + # install tools + - name: Install helm + id: helm + uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 with: - build-cache-key: run-conformance - - name: Install kind - shell: bash - run: | - # For AMD64 / x86_64 - [ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - # For ARM64 - [ $(uname -m) = aarch64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-arm64 - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind + token: ${{ secrets.GITHUB_TOKEN }} + - name: Install chainsaw + uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 + with: + release: v0.0.8 + # create cluster - name: Create kind cluster - shell: bash - run: | - set -e - export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }} - export KIND_CONFIG=vap-v1alpha1 - make kind-create-cluster + uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 + with: + node_image: kindest/node:${{ matrix.k8s-version.version }} + cluster_name: kind + config: ./scripts/config/kind/vap-v1alpha1.yaml + # deploy kyverno - name: Download kyverno images archive uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: @@ -221,26 +213,25 @@ jobs: shell: bash run: | set -e - make kind-load-image-archive + kind load image-archive kyverno.tar --name kind - name: Install kyverno shell: bash run: | set -e + export HELM=${{ steps.helm.outputs.helm-path }} export USE_CONFIG=${{ join(matrix.config.values, ',') }} make kind-install-kyverno - name: Wait for kyverno ready uses: ./.github/actions/kyverno-wait-ready - - name: Install Chainsaw - uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 - with: - release: v0.0.8 + # run tests - name: Test with Chainsaw shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e - chainsaw test --config ./test/conformance/chainsaw/_config/common.yaml --test-dir ./test/conformance/chainsaw/${{ matrix.tests }} --no-color=false + cd ./test/conformance/chainsaw && chainsaw test --test-dir ./${{ matrix.tests }} + # debug - name: Debug failure if: failure() uses: ./.github/actions/kyverno-logs @@ -267,28 +258,24 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Setup caches - uses: ./.github/actions/setup-caches - timeout-minutes: 5 - continue-on-error: true + # install tools + - name: Install helm + id: helm + uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 with: - build-cache-key: run-conformance - - name: Install kind - shell: bash - run: | - # For AMD64 / x86_64 - [ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - # For ARM64 - [ $(uname -m) = aarch64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-arm64 - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind + token: ${{ secrets.GITHUB_TOKEN }} + - name: Install chainsaw + uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 + with: + release: v0.0.8 + # create cluster - name: Create kind cluster - shell: bash - run: | - set -e - export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }} - export KIND_CONFIG=vap-v1beta1 - make kind-create-cluster + uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 + with: + node_image: kindest/node:${{ matrix.k8s-version.version }} + cluster_name: kind + config: ./scripts/config/kind/vap-v1beta1.yaml + # deploy kyverno - name: Download kyverno images archive uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: @@ -297,26 +284,25 @@ jobs: shell: bash run: | set -e - make kind-load-image-archive + kind load image-archive kyverno.tar --name kind - name: Install kyverno shell: bash run: | set -e + export HELM=${{ steps.helm.outputs.helm-path }} export USE_CONFIG=${{ join(matrix.config.values, ',') }} make kind-install-kyverno - name: Wait for kyverno ready uses: ./.github/actions/kyverno-wait-ready - - name: Install Chainsaw - uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 - with: - release: v0.0.8 + # run tests - name: Test with Chainsaw shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e - chainsaw test --config ./test/conformance/chainsaw/_config/common.yaml --test-dir ./test/conformance/chainsaw/${{ matrix.tests }} --no-color=false + cd ./test/conformance/chainsaw && chainsaw test --test-dir ./${{ matrix.tests }} + # debug - name: Debug failure if: failure() uses: ./.github/actions/kyverno-logs @@ -345,28 +331,24 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Setup caches - uses: ./.github/actions/setup-caches - timeout-minutes: 5 - continue-on-error: true + # install tools + - name: Install helm + id: helm + uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 with: - build-cache-key: run-conformance - - name: Install kind - shell: bash - run: | - # For AMD64 / x86_64 - [ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - # For ARM64 - [ $(uname -m) = aarch64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-arm64 - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind + token: ${{ secrets.GITHUB_TOKEN }} + - name: Install chainsaw + uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 + with: + release: v0.0.8 + # create cluster - name: Create kind cluster - shell: bash - run: | - set -e - export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }} - export KIND_CONFIG=vap-v1alpha1 - make kind-create-cluster + uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 + with: + node_image: kindest/node:${{ matrix.k8s-version.version }} + cluster_name: kind + config: ./scripts/config/kind/vap-v1alpha1.yaml + # deploy kyverno - name: Download kyverno images archive uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: @@ -375,26 +357,25 @@ jobs: shell: bash run: | set -e - make kind-load-image-archive + kind load image-archive kyverno.tar --name kind - name: Install kyverno shell: bash run: | set -e + export HELM=${{ steps.helm.outputs.helm-path }} export USE_CONFIG=${{ join(matrix.config.values, ',') }} make kind-install-kyverno - name: Wait for kyverno ready uses: ./.github/actions/kyverno-wait-ready - - name: Install Chainsaw - uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 - with: - release: v0.0.8 + # run tests - name: Test with Chainsaw shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e - chainsaw test --config ./test/conformance/chainsaw/_config/common.yaml --test-dir ./test/conformance/chainsaw/${{ matrix.tests }} --no-color=false + cd ./test/conformance/chainsaw && chainsaw test --test-dir ./${{ matrix.tests }} + # debug - name: Debug failure if: failure() uses: ./.github/actions/kyverno-logs @@ -421,28 +402,24 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Setup caches - uses: ./.github/actions/setup-caches - timeout-minutes: 5 - continue-on-error: true + # install tools + - name: Install helm + id: helm + uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 with: - build-cache-key: run-conformance - - name: Install kind - shell: bash - run: | - # For AMD64 / x86_64 - [ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - # For ARM64 - [ $(uname -m) = aarch64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-arm64 - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind + token: ${{ secrets.GITHUB_TOKEN }} + - name: Install chainsaw + uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 + with: + release: v0.0.8 + # create cluster - name: Create kind cluster - shell: bash - run: | - set -e - export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }} - export KIND_CONFIG=vap-v1beta1 - make kind-create-cluster + uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 + with: + node_image: kindest/node:${{ matrix.k8s-version.version }} + cluster_name: kind + config: ./scripts/config/kind/vap-v1beta1.yaml + # deploy kyverno - name: Download kyverno images archive uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: @@ -451,26 +428,25 @@ jobs: shell: bash run: | set -e - make kind-load-image-archive + kind load image-archive kyverno.tar --name kind - name: Install kyverno shell: bash run: | set -e + export HELM=${{ steps.helm.outputs.helm-path }} export USE_CONFIG=${{ join(matrix.config.values, ',') }} make kind-install-kyverno - name: Wait for kyverno ready uses: ./.github/actions/kyverno-wait-ready - - name: Install Chainsaw - uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 - with: - release: v0.0.8 + # run tests - name: Test with Chainsaw shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e - chainsaw test --config ./test/conformance/chainsaw/_config/common.yaml --test-dir ./test/conformance/chainsaw/${{ matrix.tests }} --no-color=false + cd ./test/conformance/chainsaw && chainsaw test --test-dir ./${{ matrix.tests }} + # debug - name: Debug failure if: failure() uses: ./.github/actions/kyverno-logs @@ -504,27 +480,24 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Setup caches - uses: ./.github/actions/setup-caches - timeout-minutes: 5 - continue-on-error: true + # install tools + - name: Install helm + id: helm + uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 with: - build-cache-key: run-conformance - - name: Install kind - shell: bash - run: | - # For AMD64 / x86_64 - [ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - # For ARM64 - [ $(uname -m) = aarch64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-arm64 - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind + token: ${{ secrets.GITHUB_TOKEN }} + - name: Install chainsaw + uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 + with: + release: v0.0.8 + # create cluster - name: Create kind cluster - shell: bash - run: | - set -e - export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }} - make kind-create-cluster + uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 + with: + node_image: kindest/node:${{ matrix.k8s-version.version }} + cluster_name: kind + config: ./scripts/config/kind/default.yaml + # deploy kyverno - name: Download kyverno images archive uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: @@ -533,26 +506,25 @@ jobs: shell: bash run: | set -e - make kind-load-image-archive + kind load image-archive kyverno.tar --name kind - name: Install kyverno shell: bash run: | set -e + export HELM=${{ steps.helm.outputs.helm-path }} export USE_CONFIG=${{ join(matrix.config.values, ',') }} make kind-install-kyverno - name: Wait for kyverno ready uses: ./.github/actions/kyverno-wait-ready - - name: Install Chainsaw - uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 - with: - release: v0.0.8 + # run tests - name: Test with Chainsaw shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e - chainsaw test --config ./test/conformance/chainsaw/_config/common.yaml --test-dir ./test/conformance/chainsaw/${{ matrix.tests }} --no-color=false + cd ./test/conformance/chainsaw && chainsaw test --test-dir ./${{ matrix.tests }} + # debug - name: Debug failure if: failure() uses: ./.github/actions/kyverno-logs @@ -585,27 +557,24 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Setup caches - uses: ./.github/actions/setup-caches - timeout-minutes: 5 - continue-on-error: true + # install tools + - name: Install helm + id: helm + uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 with: - build-cache-key: run-conformance - - name: Install kind - shell: bash - run: | - # For AMD64 / x86_64 - [ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - # For ARM64 - [ $(uname -m) = aarch64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-arm64 - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind + token: ${{ secrets.GITHUB_TOKEN }} + - name: Install chainsaw + uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 + with: + release: v0.0.8 + # create cluster - name: Create kind cluster - shell: bash - run: | - set -e - export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }} - make kind-create-cluster + uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 + with: + node_image: kindest/node:${{ matrix.k8s-version.version }} + cluster_name: kind + config: ./scripts/config/kind/default.yaml + # deploy kyverno - name: Download kyverno images archive uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: @@ -614,26 +583,25 @@ jobs: shell: bash run: | set -e - make kind-load-image-archive + kind load image-archive kyverno.tar --name kind - name: Install kyverno shell: bash run: | set -e + export HELM=${{ steps.helm.outputs.helm-path }} export USE_CONFIG=${{ join(matrix.config.values, ',') }} make kind-install-kyverno - name: Wait for kyverno ready uses: ./.github/actions/kyverno-wait-ready - - name: Install Chainsaw - uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 - with: - release: v0.0.8 + # run tests - name: Test with Chainsaw shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e - chainsaw test --config ./test/conformance/chainsaw/_config/common.yaml --test-dir ./test/conformance/chainsaw/${{ matrix.tests }} --no-color=false + cd ./test/conformance/chainsaw && chainsaw test --test-dir ./${{ matrix.tests }} + # debug - name: Debug failure if: failure() uses: ./.github/actions/kyverno-logs @@ -733,7 +701,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e - chainsaw test --config ./test/conformance/chainsaw/_config/common.yaml --test-dir ./test/conformance/chainsaw/${{ matrix.tests }} --no-color=false + cd ./test/conformance/chainsaw && chainsaw test --test-dir ./${{ matrix.tests }} - name: Debug failure if: failure() uses: ./.github/actions/kyverno-logs @@ -765,27 +733,24 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Setup caches - uses: ./.github/actions/setup-caches - timeout-minutes: 5 - continue-on-error: true + # install tools + - name: Install helm + id: helm + uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 with: - build-cache-key: run-conformance - - name: Install kind - shell: bash - run: | - # For AMD64 / x86_64 - [ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - # For ARM64 - [ $(uname -m) = aarch64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-arm64 - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind + token: ${{ secrets.GITHUB_TOKEN }} + - name: Install chainsaw + uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 + with: + release: v0.0.8 + # create cluster - name: Create kind cluster - shell: bash - run: | - set -e - export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }} - make kind-create-cluster + uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 + with: + node_image: kindest/node:${{ matrix.k8s-version.version }} + cluster_name: kind + config: ./scripts/config/kind/default.yaml + # deploy kyverno - name: Download kyverno images archive uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: @@ -794,26 +759,25 @@ jobs: shell: bash run: | set -e - make kind-load-image-archive + kind load image-archive kyverno.tar --name kind - name: Install kyverno shell: bash run: | set -e + export HELM=${{ steps.helm.outputs.helm-path }} export USE_CONFIG=${{ join(matrix.config.values, ',') }} make kind-install-kyverno - name: Wait for kyverno ready uses: ./.github/actions/kyverno-wait-ready - - name: Install Chainsaw - uses: kyverno/chainsaw/.github/actions/install@5d6585b843b48eaf1940a8333bb828fd6a3385e3 # v0.0.8 - with: - release: v0.0.8 + # run tests - name: Test with Chainsaw shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e - chainsaw test --config ./test/conformance/chainsaw/_config/common.yaml --test-dir ./test/conformance/chainsaw/${{ matrix.tests }} --no-color=false + cd ./test/conformance/chainsaw && chainsaw test --test-dir ./${{ matrix.tests }} + # debug - name: Debug failure if: failure() uses: ./.github/actions/kyverno-logs @@ -929,7 +893,7 @@ jobs: - name: Install CRDs run: | set -e - kubectl apply -f ./policies/CRDs + kubectl apply -f ./policies/.chainsaw/crds - name: Test with Chainsaw env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -1017,7 +981,7 @@ jobs: KYVERNO_EXPERIMENTAL=true kyverno fix test ./test/cli --save --compress make verify-cli-tests - required: + conformance-required: needs: - standard - ttl diff --git a/Makefile b/Makefile index d847b5cd16..a33c3cd66b 100644 --- a/Makefile +++ b/Makefile @@ -32,33 +32,33 @@ USE_CONFIG ?= standard # TOOLS # ######### -TOOLS_DIR := $(PWD)/.tools -KIND := $(TOOLS_DIR)/kind -KIND_VERSION := v0.20.0 -CONTROLLER_GEN := $(TOOLS_DIR)/controller-gen -CONTROLLER_GEN_VERSION := v0.12.0 -CLIENT_GEN := $(TOOLS_DIR)/client-gen -LISTER_GEN := $(TOOLS_DIR)/lister-gen -INFORMER_GEN := $(TOOLS_DIR)/informer-gen -OPENAPI_GEN := $(TOOLS_DIR)/openapi-gen -REGISTER_GEN := $(TOOLS_DIR)/register-gen -DEEPCOPY_GEN := $(TOOLS_DIR)/deepcopy-gen -DEFAULTER_GEN := $(TOOLS_DIR)/defaulter-gen -APPLYCONFIGURATION_GEN := $(TOOLS_DIR)/applyconfiguration-gen -CODE_GEN_VERSION := v0.28.0 -GEN_CRD_API_REFERENCE_DOCS := $(TOOLS_DIR)/gen-crd-api-reference-docs -GEN_CRD_API_REFERENCE_DOCS_VERSION := latest -GO_ACC := $(TOOLS_DIR)/go-acc -GO_ACC_VERSION := latest -GOIMPORTS := $(TOOLS_DIR)/goimports -GOIMPORTS_VERSION := latest -HELM := $(TOOLS_DIR)/helm -HELM_VERSION := v3.12.3 -HELM_DOCS := $(TOOLS_DIR)/helm-docs -HELM_DOCS_VERSION := v1.11.0 -KO := $(TOOLS_DIR)/ko -KO_VERSION := v0.14.1 -KUBE_VERSION := v1.25.0 +TOOLS_DIR ?= $(PWD)/.tools +KIND ?= $(TOOLS_DIR)/kind +KIND_VERSION ?= v0.20.0 +CONTROLLER_GEN ?= $(TOOLS_DIR)/controller-gen +CONTROLLER_GEN_VERSION ?= v0.12.0 +CLIENT_GEN ?= $(TOOLS_DIR)/client-gen +LISTER_GEN ?= $(TOOLS_DIR)/lister-gen +INFORMER_GEN ?= $(TOOLS_DIR)/informer-gen +OPENAPI_GEN ?= $(TOOLS_DIR)/openapi-gen +REGISTER_GEN ?= $(TOOLS_DIR)/register-gen +DEEPCOPY_GEN ?= $(TOOLS_DIR)/deepcopy-gen +DEFAULTER_GEN ?= $(TOOLS_DIR)/defaulter-gen +APPLYCONFIGURATION_GEN ?= $(TOOLS_DIR)/applyconfiguration-gen +CODE_GEN_VERSION ?= v0.28.0 +GEN_CRD_API_REFERENCE_DOCS ?= $(TOOLS_DIR)/gen-crd-api-reference-docs +GEN_CRD_API_REFERENCE_DOCS_VERSION ?= latest +GO_ACC ?= $(TOOLS_DIR)/go-acc +GO_ACC_VERSION ?= latest +GOIMPORTS ?= $(TOOLS_DIR)/goimports +GOIMPORTS_VERSION ?= latest +HELM ?= $(TOOLS_DIR)/helm +HELM_VERSION ?= v3.12.3 +HELM_DOCS ?= $(TOOLS_DIR)/helm-docs +HELM_DOCS_VERSION ?= v1.11.0 +KO ?= $(TOOLS_DIR)/ko +KO_VERSION ?= v0.14.1 +KUBE_VERSION ?= v1.25.0 TOOLS := $(KIND) $(CONTROLLER_GEN) $(CLIENT_GEN) $(LISTER_GEN) $(INFORMER_GEN) $(OPENAPI_GEN) $(REGISTER_GEN) $(DEEPCOPY_GEN) $(DEFAULTER_GEN) $(APPLYCONFIGURATION_GEN) $(GEN_CRD_API_REFERENCE_DOCS) $(GO_ACC) $(GOIMPORTS) $(HELM) $(HELM_DOCS) $(KO) ifeq ($(GOOS), darwin) SED := gsed diff --git a/test/conformance/chainsaw/_config/common.yaml b/test/conformance/chainsaw/.chainsaw.yaml similarity index 100% rename from test/conformance/chainsaw/_config/common.yaml rename to test/conformance/chainsaw/.chainsaw.yaml diff --git a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/failure-policy-test-noconfigmap-diffimage-success/chainsaw-test.yaml b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/failure-policy-test-noconfigmap-diffimage-success/chainsaw-test.yaml index fb4cc21dab..be39aaf6cf 100755 --- a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/failure-policy-test-noconfigmap-diffimage-success/chainsaw-test.yaml +++ b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/failure-policy-test-noconfigmap-diffimage-success/chainsaw-test.yaml @@ -4,6 +4,8 @@ metadata: creationTimestamp: null name: failure-policy-test-noconfigmap-diffimage-success spec: + timeouts: + delete: 2m steps: - name: step-01 try: diff --git a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/keyed-basic-namespace-selector/chainsaw-test.yaml b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/keyed-basic-namespace-selector/chainsaw-test.yaml index dd41ee132b..a0698cb902 100755 --- a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/keyed-basic-namespace-selector/chainsaw-test.yaml +++ b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/keyed-basic-namespace-selector/chainsaw-test.yaml @@ -4,6 +4,8 @@ metadata: creationTimestamp: null name: keyed-basic-namespace-selector spec: + timeouts: + delete: 2m steps: - name: step-01 try: diff --git a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/keyed-basic/chainsaw-test.yaml b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/keyed-basic/chainsaw-test.yaml index 721355fe07..59c98f02a0 100755 --- a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/keyed-basic/chainsaw-test.yaml +++ b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/keyed-basic/chainsaw-test.yaml @@ -4,6 +4,8 @@ metadata: creationTimestamp: null name: keyed-basic spec: + timeouts: + delete: 2m steps: - name: step-01 try: diff --git a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/keyed-secret/chainsaw-test.yaml b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/keyed-secret/chainsaw-test.yaml index cf4a48f98a..18a1f53bd8 100755 --- a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/keyed-secret/chainsaw-test.yaml +++ b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/keyed-secret/chainsaw-test.yaml @@ -4,6 +4,8 @@ metadata: creationTimestamp: null name: keyed-secret spec: + timeouts: + delete: 2m steps: - name: step-01 try: diff --git a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/noconfigmap-diffimage-success/chainsaw-test.yaml b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/noconfigmap-diffimage-success/chainsaw-test.yaml index 342c2da634..2bd43b29e0 100755 --- a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/noconfigmap-diffimage-success/chainsaw-test.yaml +++ b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/noconfigmap-diffimage-success/chainsaw-test.yaml @@ -4,6 +4,8 @@ metadata: creationTimestamp: null name: noconfigmap-diffimage-success spec: + timeouts: + delete: 2m steps: - name: step-01 try: diff --git a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/notary-attestation-verification/chainsaw-test.yaml b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/notary-attestation-verification/chainsaw-test.yaml index ec7903a618..6827e4aeeb 100755 --- a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/notary-attestation-verification/chainsaw-test.yaml +++ b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/notary-attestation-verification/chainsaw-test.yaml @@ -4,6 +4,8 @@ metadata: creationTimestamp: null name: notary-attestation-verification spec: + timeouts: + delete: 2m steps: - name: step-01 try: diff --git a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/notary-image-verification-secret-from-policy/chainsaw-test.yaml b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/notary-image-verification-secret-from-policy/chainsaw-test.yaml index 6e4d789247..24c1e8bfb4 100755 --- a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/notary-image-verification-secret-from-policy/chainsaw-test.yaml +++ b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/notary-image-verification-secret-from-policy/chainsaw-test.yaml @@ -4,6 +4,8 @@ metadata: creationTimestamp: null name: notary-image-verification-secret-from-policy spec: + timeouts: + delete: 2m steps: - name: step-01 try: diff --git a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/notary-image-verification/chainsaw-test.yaml b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/notary-image-verification/chainsaw-test.yaml index 88e7874d52..4c2244c0aa 100755 --- a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/notary-image-verification/chainsaw-test.yaml +++ b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/notary-image-verification/chainsaw-test.yaml @@ -4,6 +4,8 @@ metadata: creationTimestamp: null name: notary-image-verification spec: + timeouts: + delete: 2m steps: - name: step-01 try: diff --git a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/rollback-image-verification/chainsaw-test.yaml b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/rollback-image-verification/chainsaw-test.yaml index 5e08150766..1725deb689 100755 --- a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/rollback-image-verification/chainsaw-test.yaml +++ b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/rollback-image-verification/chainsaw-test.yaml @@ -4,6 +4,8 @@ metadata: creationTimestamp: null name: rollback-image-verification spec: + timeouts: + delete: 2m steps: - name: step-01 try: @@ -19,10 +21,16 @@ spec: file: deployment_new.yaml - assert: file: deployment-assert.yaml + finally: + - sleep: + duration: 5s - name: step-03 try: - script: content: kubectl -n verify-images rollout undo deployment nginx-deployment + finally: + - sleep: + duration: 5s - name: step-04 try: - assert: