chore: run force-failure-policy-ignore test using chainsaw (#8966)

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>

.github/workflows/conformance.yaml

@@ -388,6 +388,82 @@ jobs:
         if: failure()
         uses: ./.github/actions/kyverno-logs
 
+  chainsaw-force-failure-policy-ignore:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: read
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - name: force-failure-policy-ignore
+            values:
+              - standard
+              - force-failure-policy-ignore
+        k8s-version:
+          - name: v1.25
+            version: v1.25.11
+          - name: v1.26
+            version: v1.26.6
+          - name: v1.27
+            version: v1.27.3
+          - name: v1.28
+            version: v1.28.0
+        tests:
+          - force-failure-policy-ignore
+          - rbac
+    needs: prepare-images
+    name: chainsaw - ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Setup caches
+        uses: ./.github/actions/setup-caches
+        timeout-minutes: 5
+        continue-on-error: true
+        with:
+          build-cache-key: run-conformance
+      - name: Setup build env
+        uses: ./.github/actions/setup-build-env
+        timeout-minutes: 10
+      - name: Create kind cluster
+        shell: bash
+        run: |
+          set -e
+          export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
+          make kind-create-cluster
+      - name: Download kyverno images archive
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: kyverno.tar
+      - name: Load kyverno images archive in kind cluster
+        shell: bash
+        run: |
+          set -e
+          make kind-load-image-archive
+      - name: Install kyverno
+        shell: bash
+        run: |
+          set -e
+          export USE_CONFIG=${{ join(matrix.config.values, ',') }}
+          make kind-install-kyverno
+      - name: Wait for kyverno ready
+        uses: ./.github/actions/kyverno-wait-ready
+      - name: Install Chainsaw
+        uses: kyverno/chainsaw/.github/actions/install@704abd5ea8fd74189e1192733a879a00a7d527f5 # main
+        with:
+          release: v0.0.6-alpha.1
+      - name: Test with Chainsaw
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -e
+          chainsaw test  --config ./test/conformance/chainsaw/_config/common.yaml --test-dir ./test/conformance/chainsaw/${{ matrix.tests }} --no-color=false
+      - name: Debug failure
+        if: failure()
+        uses: ./.github/actions/kyverno-logs
+
   # runs conformance test suites with configuration:
   ttl:
     runs-on: ubuntu-latest

test/conformance/chainsaw/force-failure-policy-ignore/cluster-policy/fail/01-policy.yaml

@@ -0,0 +1,10 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: policy
+spec:
+  try:
+  - apply:
+      file: policy.yaml
+  - assert:
+      file: policy.yaml

test/conformance/chainsaw/force-failure-policy-ignore/cluster-policy/fail/02-webhooks.yaml

@@ -0,0 +1,8 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: webhooks
+spec:
+  try:
+  - assert:
+      file: webhooks-assert.yaml

test/conformance/chainsaw/force-failure-policy-ignore/cluster-policy/fail/README.md

@@ -0,0 +1,7 @@
+## Description
+
+This test creates a policy with `failurePolicy: Fail` but the configuration has `forceWebhookFailurePolicyIgnore: true`.
+
+## Expected Behavior
+
+Webhooks should be configured with `failurePolicy: Ignore` regardless of the failure policy configured in the policies.

test/conformance/chainsaw/force-failure-policy-ignore/cluster-policy/fail/policy-assert.yaml

@@ -0,0 +1,19 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+status:
+  conditions:
+    - reason: Succeeded
+      status: "True"
+      type: Ready
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: add-labels
+status:
+  conditions:
+    - reason: Succeeded
+      status: "True"
+      type: Ready

test/conformance/chainsaw/force-failure-policy-ignore/cluster-policy/fail/policy.yaml

@@ -0,0 +1,47 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+spec:
+  failurePolicy: Fail
+  validationFailureAction: Enforce
+  background: false
+  rules:
+    - name: require-team
+      match:
+        any:
+          - resources:
+              kinds:
+                - Pod
+      validate:
+        message: 'The label `team` is required.'
+        pattern:
+          metadata:
+            labels:
+              team: '?*'
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: add-labels
+spec:
+  failurePolicy: Fail
+  validationFailureAction: Enforce
+  background: false
+  rules:
+  - name: add-labels
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+          - Service
+          - ConfigMap
+          - Secret
+    mutate:
+      patchStrategicMerge:
+        metadata:
+          labels:
+            foo: bar

test/conformance/chainsaw/force-failure-policy-ignore/cluster-policy/fail/webhooks-assert.yaml

@@ -0,0 +1,39 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    webhook.kyverno.io/managed-by: kyverno
+  name: kyverno-resource-validating-webhook-cfg
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: kyverno-svc
+      namespace: kyverno
+      path: /validate/ignore
+      port: 443
+  failurePolicy: Ignore
+  matchPolicy: Equivalent
+  name: validate.kyverno.svc-ignore
+  sideEffects: NoneOnDryRun
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  labels:
+    webhook.kyverno.io/managed-by: kyverno
+  name: kyverno-resource-mutating-webhook-cfg
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: kyverno-svc
+      namespace: kyverno
+      path: /mutate/ignore
+      port: 443
+  failurePolicy: Ignore
+  matchPolicy: Equivalent
+  name: mutate.kyverno.svc-ignore
+  sideEffects: NoneOnDryRun