chore: run tests with chainsaw (#8762)

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

.github/workflows/conformance.yaml

@@ -156,6 +156,99 @@ jobs:
         if: failure()
         uses: ./.github/actions/kyverno-logs
 
+  chainsaw:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: read
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - name: standard
+            values:
+              - standard
+        k8s-version:
+          - name: v1.25
+            version: v1.25.11
+          - name: v1.26
+            version: v1.26.6
+          - name: v1.27
+            version: v1.27.3
+          - name: v1.28
+            version: v1.28.0
+        tests:
+          # - autogen
+          # - background-only
+          # - cleanup
+          # - deferred
+          # - events
+          # - exceptions
+          # - filter
+          # - generate/clusterpolicy
+          # - generate/policy
+          # - generate/validation
+          # - mutate
+          # - policy-validation
+          # - rangeoperators
+          # - rbac
+          # - reports
+          # - validate
+          # - verify-manifests
+          # - verifyImages
+          - webhooks
+    needs: prepare-images
+    name: chainsaw - ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Setup caches
+        uses: ./.github/actions/setup-caches
+        timeout-minutes: 5
+        continue-on-error: true
+        with:
+          build-cache-key: run-conformance
+      - name: Setup build env
+        uses: ./.github/actions/setup-build-env
+        timeout-minutes: 10
+      - name: Create kind cluster
+        shell: bash
+        run: |
+          set -e
+          export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
+          make kind-create-cluster
+      - name: Download kyverno images archive
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: kyverno.tar
+      - name: Load kyverno images archive in kind cluster
+        shell: bash
+        run: |
+          set -e
+          make kind-load-image-archive
+      - name: Install kyverno
+        shell: bash
+        run: |
+          set -e
+          export USE_CONFIG=${{ join(matrix.config.values, ',') }}
+          make kind-install-kyverno
+      - name: Wait for kyverno ready
+        uses: ./.github/actions/kyverno-wait-ready
+      - name: Install Chainsaw
+        shell: bash
+        run: |
+          set -e
+          go install github.com/kyverno/chainsaw@ffa5508c1d0fbba24d70a3a51ef777392a9b32b6
+      - name: Test with Chainsaw
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -e
+          chainsaw test  --config ./test/conformance/chainsaw/_config/common.yaml --test-dir ./test/conformance/chainsaw/${{ matrix.tests }} --no-color=false
+      - name: Debug failure
+        if: failure()
+        uses: ./.github/actions/kyverno-logs
+
   # runs conformance test suites with configuration:
   ttl:
     runs-on: ubuntu-latest

test/conformance/chainsaw/_config/common.yaml

@@ -0,0 +1,10 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Configuration
+metadata:
+  name: congiguration
+spec:
+  parallel: 1
+  timeout: 1m30s
+  fullName: true
+  failFast: true
+  excludeTestRegex: '_.+'

test/conformance/chainsaw/webhooks/all-scale/01-policy.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: policy
+spec:
+  apply:
+  - file: policy.yaml
+  assert:
+  - file: policy-assert.yaml

test/conformance/chainsaw/webhooks/all-scale/02-webhooks.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: webhooks
+spec:
+  assert:
+  - file: webhooks.yaml

test/conformance/chainsaw/webhooks/all-scale/README.md

@@ -0,0 +1,9 @@
+## Description
+
+This test verifies the resource validation webhook is configured correctly when a policy targets all `*/scale` subresources.
+
+## Steps
+
+1.  - Create a policy targeting `*/scale`
+    - Assert policy gets ready
+1.  - Assert that the resource validation webhook is configured correctly

test/conformance/chainsaw/webhooks/all-scale/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+status:
+  conditions:
+    - reason: Succeeded
+      status: "True"
+      type: Ready

test/conformance/chainsaw/webhooks/all-scale/policy.yaml

@@ -0,0 +1,22 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+spec:
+  validationFailureAction: Audit
+  background: false
+  rules:
+    - name: require-team
+      match:
+        any:
+          - resources:
+              kinds:
+                - '*/scale'
+      validate:
+        message: 'The label `team` is required.'
+        pattern:
+          metadata:
+            labels:
+              team: '?*'

test/conformance/chainsaw/webhooks/all-scale/webhooks.yaml

@@ -0,0 +1,20 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    webhook.kyverno.io/managed-by: kyverno
+  name: kyverno-resource-validating-webhook-cfg
+webhooks:
+- rules:
+  - apiGroups:
+    - '*'
+    apiVersions:
+    - '*'
+    operations:
+    - CREATE
+    - UPDATE
+    - DELETE
+    - CONNECT
+    resources:
+    - '*/scale'
+    scope: '*'

test/conformance/chainsaw/webhooks/double-wildcard/01-policy.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: policy
+spec:
+  apply:
+  - file: policy.yaml
+  assert:
+  - file: policy-assert.yaml

test/conformance/chainsaw/webhooks/double-wildcard/02-webhooks.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: webhooks
+spec:
+  assert:
+  - file: webhooks.yaml

test/conformance/chainsaw/webhooks/double-wildcard/README.md

@@ -0,0 +1,9 @@
+## Description
+
+This test verifies the resource validation webhook is configured correctly when a policy targets all `*/*` resources and subresources.
+
+## Steps
+
+1.  - Create a policy targeting `*/*`
+    - Assert policy gets ready
+1.  - Assert that the resource validation webhook is configured correctly

test/conformance/chainsaw/webhooks/double-wildcard/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+status:
+  conditions:
+    - reason: Succeeded
+      status: "True"
+      type: Ready

test/conformance/chainsaw/webhooks/double-wildcard/policy.yaml

@@ -0,0 +1,22 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+spec:
+  validationFailureAction: Audit
+  background: false
+  rules:
+    - name: require-team
+      match:
+        any:
+          - resources:
+              kinds:
+                - '*/*'
+      validate:
+        message: 'The label `team` is required.'
+        pattern:
+          metadata:
+            labels:
+              team: '?*'

test/conformance/chainsaw/webhooks/double-wildcard/webhooks.yaml

@@ -0,0 +1,21 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    webhook.kyverno.io/managed-by: kyverno
+  name: kyverno-resource-validating-webhook-cfg
+webhooks:
+- failurePolicy: Fail
+  rules:
+  - apiGroups:
+    - '*'
+    apiVersions:
+    - '*'
+    operations:
+    - CREATE
+    - UPDATE
+    - DELETE
+    - CONNECT
+    resources:
+    - '*/*'
+    scope: '*'

test/conformance/chainsaw/webhooks/expected-webhooks/01-webhooks.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: webhooks
+spec:
+  assert:
+  - file: webhooks.yaml

test/conformance/chainsaw/webhooks/expected-webhooks/README.md

@@ -0,0 +1,11 @@
+## Description
+
+This test verifies expected webhooks are created.
+
+## Steps
+
+1.  - Assert webhook `kyverno-policy-validating-webhook-cfg` exists
+    - Assert webhook `kyverno-resource-validating-webhook-cfg` exists
+    - Assert webhook `kyverno-policy-mutating-webhook-cfg` exists
+    - Assert webhook `kyverno-resource-mutating-webhook-cfg` exists
+    - Assert webhook `kyverno-verify-mutating-webhook-cfg` exists

test/conformance/chainsaw/webhooks/expected-webhooks/webhooks.yaml

@@ -0,0 +1,34 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    webhook.kyverno.io/managed-by: kyverno
+  name: kyverno-policy-validating-webhook-cfg
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    webhook.kyverno.io/managed-by: kyverno
+  name: kyverno-resource-validating-webhook-cfg
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  labels:
+    webhook.kyverno.io/managed-by: kyverno
+  name: kyverno-policy-mutating-webhook-cfg
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  labels:
+    webhook.kyverno.io/managed-by: kyverno
+  name: kyverno-resource-mutating-webhook-cfg
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  labels:
+    webhook.kyverno.io/managed-by: kyverno
+  name: kyverno-verify-mutating-webhook-cfg

test/conformance/chainsaw/webhooks/only-pod/01-policy.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: policy
+spec:
+  apply:
+  - file: policy.yaml
+  assert:
+  - file: policy-assert.yaml

test/conformance/chainsaw/webhooks/only-pod/02-webhooks.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: webhooks
+spec:
+  assert:
+  - file: webhooks.yaml

test/conformance/chainsaw/webhooks/only-pod/README.md

@@ -0,0 +1,9 @@
+## Description
+
+This test verifies the resource validation webhook is configured correctly when a policy targets `Pod`.
+
+## Steps
+
+1.  - Create a policy targeting `Pod`
+    - Assert policy gets ready
+1.  - Assert that the resource validation webhook is configured correctly

test/conformance/chainsaw/webhooks/only-pod/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+status:
+  conditions:
+    - reason: Succeeded
+      status: "True"
+      type: Ready

test/conformance/chainsaw/webhooks/only-pod/policy.yaml

@@ -0,0 +1,22 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+spec:
+  validationFailureAction: Audit
+  background: false
+  rules:
+    - name: require-team
+      match:
+        any:
+          - resources:
+              kinds:
+                - Pod
+      validate:
+        message: 'The label `team` is required.'
+        pattern:
+          metadata:
+            labels:
+              team: '?*'

test/conformance/chainsaw/webhooks/only-pod/webhooks.yaml

@@ -0,0 +1,21 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    webhook.kyverno.io/managed-by: kyverno
+  name: kyverno-resource-validating-webhook-cfg
+webhooks:
+- rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    - DELETE
+    - CONNECT
+    resources:
+    - pods
+    - pods/ephemeralcontainers
+    scope: '*'

test/conformance/chainsaw/webhooks/pod-all-subresources/01-policy.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: policy
+spec:
+  apply:
+  - file: policy.yaml
+  assert:
+  - file: policy-assert.yaml

test/conformance/chainsaw/webhooks/pod-all-subresources/02-webhooks.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: webhooks
+spec:
+  assert:
+  - file: webhooks.yaml

test/conformance/chainsaw/webhooks/pod-all-subresources/README.md

@@ -0,0 +1,9 @@
+## Description
+
+This test verifies the resource validation webhook is configured correctly when a policy targets all `Pod/*` subresources.
+
+## Steps
+
+1.  - Create a policy targeting `Pod/*`
+    - Assert policy gets ready
+1.  - Assert that the resource validation webhook is configured correctly

test/conformance/chainsaw/webhooks/pod-all-subresources/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+status:
+  conditions:
+    - reason: Succeeded
+      status: "True"
+      type: Ready

test/conformance/chainsaw/webhooks/pod-all-subresources/policy.yaml

@@ -0,0 +1,22 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+spec:
+  validationFailureAction: Audit
+  background: false
+  rules:
+    - name: require-team
+      match:
+        any:
+          - resources:
+              kinds:
+                - Pod/*
+      validate:
+        message: 'The label `team` is required.'
+        pattern:
+          metadata:
+            labels:
+              team: '?*'

test/conformance/chainsaw/webhooks/pod-all-subresources/webhooks.yaml

@@ -0,0 +1,28 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    webhook.kyverno.io/managed-by: kyverno
+  name: kyverno-resource-validating-webhook-cfg
+webhooks:
+- rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    - DELETE
+    - CONNECT
+    resources:
+    - pods/attach
+    - pods/binding
+    - pods/ephemeralcontainers
+    - pods/eviction
+    - pods/exec
+    - pods/log
+    - pods/portforward
+    - pods/proxy
+    - pods/status
+    scope: '*'

test/conformance/chainsaw/webhooks/scale/01-policy.yaml

@@ -0,0 +1,8 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: policy
+spec:
+  apply:
+  - file: policy.yaml
+    shouldFail: true

test/conformance/chainsaw/webhooks/scale/README.md

@@ -0,0 +1,4 @@
+## Description
+
+This test tries to create a policy targeting the `Scale` kind.
+The `Scale` kind doesn't map to a top level resource and therefore the policy is expected to be rejected.

test/conformance/chainsaw/webhooks/scale/policy.yaml

@@ -0,0 +1,22 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+spec:
+  validationFailureAction: Audit
+  background: false
+  rules:
+    - name: require-team
+      match:
+        any:
+          - resources:
+              kinds:
+                - Scale
+      validate:
+        message: 'The label `team` is required.'
+        pattern:
+          metadata:
+            labels:
+              team: '?*'

test/conformance/chainsaw/webhooks/unknown-kind/01-unknown-kind.yaml

@@ -0,0 +1,8 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: unknown-kind
+spec:
+  apply:
+  - file: policy-1.yaml
+    shouldFail: true

test/conformance/chainsaw/webhooks/unknown-kind/02-unknown-kind-subresource.yaml

@@ -0,0 +1,8 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: unknown-kind-subresource
+spec:
+  apply:
+  - file: policy-2.yaml
+    shouldFail: true

test/conformance/chainsaw/webhooks/unknown-kind/03-wrong-version.yaml

@@ -0,0 +1,8 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: wrong-version
+spec:
+  apply:
+  - file: policy-3.yaml
+    shouldFail: true

test/conformance/chainsaw/webhooks/unknown-kind/04-unknown-subresource.yaml

@@ -0,0 +1,8 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: unknown-subresource
+spec:
+  apply:
+  - file: policy-4.yaml
+    shouldFail: true

test/conformance/chainsaw/webhooks/unknown-kind/README.md

@@ -0,0 +1,4 @@
+## Description
+
+This test tries to create policies with different combinations of unknown kind and/or subresource.
+The policies should be rejected.

test/conformance/chainsaw/webhooks/unknown-kind/policy-1.yaml

@@ -0,0 +1,20 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: unknown
+spec:
+  validationFailureAction: Audit
+  background: false
+  rules:
+    - name: unknown
+      match:
+        any:
+          - resources:
+              kinds:
+                - Foo
+      validate:
+        message: 'The label `team` is required.'
+        pattern:
+          metadata:
+            labels:
+              team: '?*'

test/conformance/chainsaw/webhooks/unknown-kind/policy-2.yaml

@@ -0,0 +1,20 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: unknown
+spec:
+  validationFailureAction: Audit
+  background: false
+  rules:
+    - name: unknown
+      match:
+        any:
+          - resources:
+              kinds:
+                - Foo/*
+      validate:
+        message: 'The label `team` is required.'
+        pattern:
+          metadata:
+            labels:
+              team: '?*'

test/conformance/chainsaw/webhooks/unknown-kind/policy-3.yaml

@@ -0,0 +1,20 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: unknown
+spec:
+  validationFailureAction: Audit
+  background: false
+  rules:
+    - name: unknown
+      match:
+        any:
+          - resources:
+              kinds:
+                - v2/Pod
+      validate:
+        message: 'The label `team` is required.'
+        pattern:
+          metadata:
+            labels:
+              team: '?*'

test/conformance/chainsaw/webhooks/unknown-kind/policy-4.yaml

@@ -0,0 +1,20 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: unknown
+spec:
+  validationFailureAction: Audit
+  background: false
+  rules:
+    - name: unknown
+      match:
+        any:
+          - resources:
+              kinds:
+                - Pod/foo
+      validate:
+        message: 'The label `team` is required.'
+        pattern:
+          metadata:
+            labels:
+              team: '?*'

test/conformance/chainsaw/webhooks/wildcard/01-policy.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: policy
+spec:
+  apply:
+  - file: policy.yaml
+  assert:
+  - file: policy-assert.yaml

test/conformance/chainsaw/webhooks/wildcard/02-webhooks.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: webhooks
+spec:
+  assert:
+  - file: webhooks.yaml

test/conformance/chainsaw/webhooks/wildcard/README.md

@@ -0,0 +1,9 @@
+## Description
+
+This test verifies the resource validation webhook is configured correctly when a policy targets all `*` resources.
+
+## Steps
+
+1.  - Create a policy targeting `*`
+    - Assert policy gets ready
+1.  - Assert that the resource validation webhook is configured correctly

test/conformance/chainsaw/webhooks/wildcard/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+status:
+  conditions:
+    - reason: Succeeded
+      status: "True"
+      type: Ready

test/conformance/chainsaw/webhooks/wildcard/policy.yaml

@@ -0,0 +1,22 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+spec:
+  validationFailureAction: Audit
+  background: false
+  rules:
+    - name: require-team
+      match:
+        any:
+          - resources:
+              kinds:
+                - '*'
+      validate:
+        message: 'The label `team` is required.'
+        pattern:
+          metadata:
+            labels:
+              team: '?*'

test/conformance/chainsaw/webhooks/wildcard/webhooks.yaml

@@ -0,0 +1,22 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    webhook.kyverno.io/managed-by: kyverno
+  name: kyverno-resource-validating-webhook-cfg
+webhooks:
+- failurePolicy: Fail
+  rules:
+  - apiGroups:
+    - '*'
+    apiVersions:
+    - '*'
+    operations:
+    - CREATE
+    - UPDATE
+    - DELETE
+    - CONNECT
+    resources:
+    - '*'
+    - pods/ephemeralcontainers
+    scope: '*'