chore: run conformance tests with multiple configs (#6811)

* chore: run conformance tests with multiple configs

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* kuttl

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

.github/workflows/conformance.yaml

@@ -30,6 +30,8 @@ jobs:
           retention-days: 1
           if-no-files-found: error
 
+  # runs conformance test suites with configuration:
+  # - standard
   run-conformance:
     strategy:
       fail-fast: false
@@ -83,3 +85,50 @@ jobs:
       - name: Debug failure
         if: failure()
         uses: ./.github/actions/kyverno-logs
+
+  # runs conformance test suites with configuration:
+  # - standard
+  # - force-failure-policy-ignore
+  force-failure-policy-ignore:
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s-version:
+          - v1.24.7
+          - v1.25.3
+          - v1.26.0
+        tests:
+          - force-failure-policy-ignore
+          - rbac
+    runs-on: ubuntu-latest
+    needs: prepare-images
+    steps:
+      - name: Checkout
+        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
+      - name: Setup build env
+        uses: ./.github/actions/setup-build-env
+        with:
+          build-cache-key: run-conformance
+      - name: Create kind cluster
+        run: |
+          export KIND_IMAGE=kindest/node:${{ matrix.k8s-version }}
+          make kind-create-cluster
+      - name: Download kyverno images archive
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: kyverno.tar
+      - name: Load kyverno images archive in kind cluster
+        run: make kind-load-image-archive
+      - name: Install kyverno
+        run: |
+          export USE_CONFIG=standard,force-failure-policy-ignore
+          make kind-install-kyverno
+      - name: Wait for kyverno ready
+        uses: ./.github/actions/kyverno-wait-ready
+      - name: Test with kuttl
+        run: |
+          ./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }} \
+            --config ./test/conformance/kuttl/_config/common.yaml
+      - name: Debug failure
+        if: failure()
+        uses: ./.github/actions/kyverno-logs

test/conformance/kuttl/force-failure-policy-ignore/cluster-policy/fail/01-policy.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- policy.yaml
+assert:
+- policy-assert.yaml

test/conformance/kuttl/force-failure-policy-ignore/cluster-policy/fail/02-webhooks.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+assert:
+- webhooks-assert.yaml

test/conformance/kuttl/force-failure-policy-ignore/cluster-policy/fail/README.md

@@ -0,0 +1,7 @@
+## Description
+
+This test creates a policy with `failurePolicy: Fail` but the configuration has `forceWebhookFailurePolicyIgnore: true`.
+
+## Expected Behavior
+
+Webhooks should be configured with `failurePolicy: Ignore` regardless of the failure policy configured in the policies.

test/conformance/kuttl/force-failure-policy-ignore/cluster-policy/fail/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+status:
+  conditions:
+    - reason: Succeeded
+      status: "True"
+      type: Ready

test/conformance/kuttl/force-failure-policy-ignore/cluster-policy/fail/policy.yaml

@@ -0,0 +1,23 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+spec:
+  failurePolicy: Fail
+  validationFailureAction: Enforce
+  background: false
+  rules:
+    - name: require-team
+      match:
+        any:
+          - resources:
+              kinds:
+                - Pod
+      validate:
+        message: 'The label `team` is required.'
+        pattern:
+          metadata:
+            labels:
+              team: '?*'

test/conformance/kuttl/force-failure-policy-ignore/cluster-policy/fail/webhooks-assert.yaml

@@ -0,0 +1,33 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    webhook.kyverno.io/managed-by: kyverno
+  name: kyverno-resource-validating-webhook-cfg
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: kyverno-svc
+      namespace: kyverno
+      path: /mutate/ignore
+      port: 443
+  failurePolicy: Ignore
+  matchPolicy: Equivalent
+  name: mutate.kyverno.svc-ignore
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    - DELETE
+    - CONNECT
+    resources:
+    - pods
+    - pods/ephemeralcontainers
+    scope: '*'
+  sideEffects: NoneOnDryRun