chore: add chainsaw tests for policy based webhook configuration (#9561)

* add chainsaw tests

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* add chainsaw tests

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* update policy

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* add delay

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* add debug steps

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* attempt fix

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* attempt fix

Signed-off-by: ShutingZhao <shuting@nirmata.com>

---------

Signed-off-by: ShutingZhao <shuting@nirmata.com>

.github/workflows/conformance.yaml

@@ -234,7 +234,7 @@ jobs:
         if: failure()
         uses: ./.github/actions/kyverno-logs
 
-  validatingadmissionpolicies-v1beta1:
+  k8s-version-specific-tests-above-1-28:
     runs-on: ubuntu-latest
     permissions:
       packages: read
@@ -253,6 +253,7 @@ jobs:
             version: v1.29.0
         tests:
           - generate-validating-admission-policy
+          - webhook-configurations
     needs: prepare-images
     name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
     steps:
@@ -972,7 +973,7 @@ jobs:
     - ttl
     - force-failure-policy-ignore
     - validatingadmissionpolicies-v1alpha1
-    - validatingadmissionpolicies-v1beta1
+    - k8s-version-specific-tests-above-1-28
     - validatingadmissionpolicies-reports-v1alpha1
     - validatingadmissionpolicies-reports-v1beta1
     - custom-sigstore
@@ -991,7 +992,7 @@ jobs:
     - ttl
     - force-failure-policy-ignore
     - validatingadmissionpolicies-v1alpha1
-    - validatingadmissionpolicies-v1beta1
+    - k8s-version-specific-tests-above-1-28
     - validatingadmissionpolicies-reports-v1alpha1
     - validatingadmissionpolicies-reports-v1beta1
     - custom-sigstore

test/conformance/chainsaw/webhook-configurations/cpol-match-conditions-block/README.md

@@ -0,0 +1,11 @@
+## Description
+
+This test checks fine-grained webhook configuration is synced to admission webhooks.
+
+## Expected Behavior
+
+The request sent from `system:masters` group should be forwarded to Kyverno and get blocked due to the violation
+
+## Reference Issue(s)
+
+#9111

test/conformance/chainsaw/webhook-configurations/cpol-match-conditions-block/chainsaw-test.yaml

@@ -0,0 +1,38 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: double-wildcard
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: ns.yaml
+    - assert:
+        file: ns.yaml
+  - name: step-02
+    try:
+    - apply:
+        file: policy.yaml
+    - assert:
+        file: policy-assert.yaml
+    - sleep:
+        duration: 3s
+  - name: step-03
+    try:
+    - command:
+        args:
+        - get
+        - validatingwebhookconfigurations
+        - kyverno-resource-validating-webhook-cfg
+        - -o
+        - yaml
+        entrypoint: kubectl
+  - name: step-04
+    try:
+    - apply:
+        expect:
+        - check:
+            ($error != null): true
+        file: pod.yaml

test/conformance/chainsaw/webhook-configurations/cpol-match-conditions-block/ns.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cpol-fine-grained-match-conditions-ns

test/conformance/chainsaw/webhook-configurations/cpol-match-conditions-block/pod.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: disallow-annotations-example
+  namespace: cpol-fine-grained-match-conditions-ns
+spec:
+  containers:
+  - name: example
+    image: busybox:latest
+    args: ["sleep", "infinity"]

test/conformance/chainsaw/webhook-configurations/cpol-match-conditions-block/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: cpol-fine-grained-match-conditions-disallow-latest-image-tag
+status:
+  conditions:
+    - reason: Succeeded
+      status: "True"
+      type: Ready

test/conformance/chainsaw/webhook-configurations/cpol-match-conditions-block/policy.yaml

@@ -0,0 +1,43 @@
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+  name: cpol-fine-grained-match-conditions-disallow-latest-image-tag
+spec:
+  admission: true
+  background: true
+  webhookConfiguration:
+    matchConditions:
+    - name: "select-namespace"
+      expression: '(object.metadata.namespace == "cpol-fine-grained-match-conditions-ns")'
+    - name: 'exclude-requests-by-groups'
+      expression: '!("system:nodes" in request.userInfo.groups)' 
+  rules:
+  - match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    name: require-image-tag
+    validate:
+      message: An image tag is required
+      pattern:
+        spec:
+          containers:
+          - image: '*:*'
+  - match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    name: validate-image-tag
+    validate:
+      message: Using a mutable image tag e.g. 'latest' is not allowed
+      pattern:
+        spec:
+          containers:
+          - image: '!*:latest'
+  validationFailureAction: Enforce
+  failurePolicy: Ignore

test/conformance/chainsaw/webhook-configurations/cpol-match-conditions-pass/README.md

@@ -0,0 +1,11 @@
+## Description
+
+This test checks fine-grained webhook configuration is synced to admission webhooks.
+
+## Expected Behavior
+
+The request sent from `system:masters` group should be forwarded to Kyverno and get blocked due to the violation
+
+## Reference Issue(s)
+
+#9111

test/conformance/chainsaw/webhook-configurations/cpol-match-conditions-pass/chainsaw-test.yaml

@@ -0,0 +1,37 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: double-wildcard
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: ns.yaml
+    - assert:
+        file: ns.yaml
+  - name: step-02
+    try:
+    - apply:
+        file: policy.yaml
+    - assert:
+        file: policy-assert.yaml
+    - sleep:
+        duration: 3s
+  - name: step-03
+    try:
+    - command:
+        args:
+        - get
+        - validatingwebhookconfigurations
+        - kyverno-resource-validating-webhook-cfg
+        - -o
+        - yaml
+        entrypoint: kubectl
+  - name: step-04
+    try:
+    - apply:
+        file: pod.yaml
+    - assert:
+        file: pod.yaml

test/conformance/chainsaw/webhook-configurations/cpol-match-conditions-pass/ns.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cpol-fine-grained-match-conditions-pass-ns

test/conformance/chainsaw/webhook-configurations/cpol-match-conditions-pass/pod.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: disallow-annotations-example
+  namespace: cpol-fine-grained-match-conditions-pass-ns
+spec:
+  containers:
+  - name: example
+    image: busybox:latest
+    args: ["sleep", "infinity"]

test/conformance/chainsaw/webhook-configurations/cpol-match-conditions-pass/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: cpol-fine-grained-match-conditions-pass
+status:
+  conditions:
+    - reason: Succeeded
+      status: "True"
+      type: Ready

test/conformance/chainsaw/webhook-configurations/cpol-match-conditions-pass/policy.yaml

@@ -0,0 +1,43 @@
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+  name: cpol-fine-grained-match-conditions-pass
+spec:
+  admission: true
+  background: true
+  webhookConfiguration:
+    matchConditions:
+    - name: "select-namespace"
+      expression: '(object.metadata.namespace == "cpol-fine-grained-match-conditions-pass-ns")'
+    - name: 'exclude-requests-by-groups'
+      expression: '!("system:authenticated" in request.userInfo.groups)' 
+  rules:
+  - match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    name: require-image-tag
+    validate:
+      message: An image tag is required
+      pattern:
+        spec:
+          containers:
+          - image: '*:*'
+  - match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    name: validate-image-tag
+    validate:
+      message: Using a mutable image tag e.g. 'latest' is not allowed
+      pattern:
+        spec:
+          containers:
+          - image: '!*:latest'
+  validationFailureAction: Enforce
+  failurePolicy: Ignore