test: add kuttl tests for background only policies (#7709)

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

.github/workflows/conformance.yaml

@@ -57,6 +57,7 @@ jobs:
             version: v1.27.1
         tests:
           - autogen
+          - background-only
           - cleanup
           - deferred
           - events

test/conformance/kuttl/background-only/cluster-policy/no-admission-event/01-policy.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- policy.yaml
+assert:
+- policy-assert.yaml

test/conformance/kuttl/background-only/cluster-policy/no-admission-event/02-resource.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- resource.yaml

test/conformance/kuttl/background-only/cluster-policy/no-admission-event/03-event.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+assert:
+- background-event.yaml
+error:
+- admission-event.yaml

test/conformance/kuttl/background-only/cluster-policy/no-admission-event/README.md

@@ -0,0 +1,10 @@
+## Description
+
+This test creates a policy with `admission` set to `false`.
+Then it creates a resource that violates the policy.
+
+## Expected Behavior
+
+The resource creates fine as the policy doesn't apply at admission time.
+No admission ezvent is created.
+One background event is created.

test/conformance/kuttl/background-only/cluster-policy/no-admission-event/admission-event.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+involvedObject:
+  apiVersion: v1
+  kind: Pod
+  name: pod
+kind: Event
+metadata: {}
+source:
+  component: kyverno-admission

test/conformance/kuttl/background-only/cluster-policy/no-admission-event/background-event.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+involvedObject:
+  apiVersion: v1
+  kind: Pod
+  name: pod
+kind: Event
+metadata: {}
+source:
+  component: kyverno-scan

test/conformance/kuttl/background-only/cluster-policy/no-admission-event/policy-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: validate
+spec: {}
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/kuttl/background-only/cluster-policy/no-admission-event/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: validate
+spec:
+  validationFailureAction: Enforce
+  admission: false
+  background: true
+  rules:
+  - name: validate
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      deny: {}

test/conformance/kuttl/background-only/cluster-policy/no-admission-event/resource.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod
+spec:
+  containers:
+    - name: container
+      image: nginx:latest
+      ports:
+        - containerPort: 80

test/conformance/kuttl/background-only/cluster-policy/no-admission-report/01-policy.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- policy.yaml
+assert:
+- policy-assert.yaml

test/conformance/kuttl/background-only/cluster-policy/no-admission-report/02-resource.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- resource.yaml

test/conformance/kuttl/background-only/cluster-policy/no-admission-report/03-report.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+error:
+- admission-report.yaml

test/conformance/kuttl/background-only/cluster-policy/no-admission-report/README.md

@@ -0,0 +1,9 @@
+## Description
+
+This test creates a policy with `admission` set to `false`.
+Then it creates a resource that violates the policy.
+
+## Expected Behavior
+
+The resource creates fine as the policy doesn't apply at admission time.
+No admission report is created.

test/conformance/kuttl/background-only/cluster-policy/no-admission-report/admission-report.yaml

@@ -0,0 +1,7 @@
+apiVersion: kyverno.io/v1alpha2
+kind: AdmissionReport
+metadata:
+  ownerReferences:
+  - apiVersion: v1
+    kind: Pod
+    name: pod

test/conformance/kuttl/background-only/cluster-policy/no-admission-report/policy-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: validate
+spec: {}
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/kuttl/background-only/cluster-policy/no-admission-report/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: validate
+spec:
+  validationFailureAction: Enforce
+  admission: false
+  background: true
+  rules:
+  - name: validate
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      deny: {}

test/conformance/kuttl/background-only/cluster-policy/no-admission-report/resource.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod
+spec:
+  containers:
+    - name: container
+      image: nginx:latest
+      ports:
+        - containerPort: 80

test/conformance/kuttl/background-only/cluster-policy/not-rejected/01-policy.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- policy.yaml
+assert:
+- policy-assert.yaml

test/conformance/kuttl/background-only/cluster-policy/not-rejected/02-resource.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- resource.yaml

test/conformance/kuttl/background-only/cluster-policy/not-rejected/README.md

@@ -0,0 +1,8 @@
+## Description
+
+This test creates a policy with `admission` set to `false`.
+Then it creates a resource that violates the policy.
+
+## Expected Behavior
+
+The resource creates fine as the policy doesn't apply at admission time.

test/conformance/kuttl/background-only/cluster-policy/not-rejected/policy-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: validate
+spec: {}
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/kuttl/background-only/cluster-policy/not-rejected/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: validate
+spec:
+  validationFailureAction: Enforce
+  admission: false
+  background: true
+  rules:
+  - name: validate
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      deny: {}

test/conformance/kuttl/background-only/cluster-policy/not-rejected/resource.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod
+spec:
+  containers:
+    - name: container
+      image: nginx:latest
+      ports:
+        - containerPort: 80

test/conformance/kuttl/background-only/policy/no-admission-event/01-policy.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- policy.yaml
+assert:
+- policy-assert.yaml

test/conformance/kuttl/background-only/policy/no-admission-event/02-resource.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- resource.yaml

test/conformance/kuttl/background-only/policy/no-admission-event/03-event.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+assert:
+- background-event.yaml
+error:
+- admission-event.yaml

test/conformance/kuttl/background-only/policy/no-admission-event/README.md

@@ -0,0 +1,10 @@
+## Description
+
+This test creates a policy with `admission` set to `false`.
+Then it creates a resource that violates the policy.
+
+## Expected Behavior
+
+The resource creates fine as the policy doesn't apply at admission time.
+No admission ezvent is created.
+One background event is created.

test/conformance/kuttl/background-only/policy/no-admission-event/admission-event.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+involvedObject:
+  apiVersion: v1
+  kind: Pod
+  name: pod
+kind: Event
+metadata: {}
+source:
+  component: kyverno-admission

test/conformance/kuttl/background-only/policy/no-admission-event/background-event.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+involvedObject:
+  apiVersion: v1
+  kind: Pod
+  name: pod
+kind: Event
+metadata: {}
+source:
+  component: kyverno-scan

test/conformance/kuttl/background-only/policy/no-admission-event/policy-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: validate
+spec: {}
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/kuttl/background-only/policy/no-admission-event/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: validate
+spec:
+  validationFailureAction: Enforce
+  admission: false
+  background: true
+  rules:
+  - name: validate
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      deny: {}

test/conformance/kuttl/background-only/policy/no-admission-event/resource.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod
+spec:
+  containers:
+    - name: container
+      image: nginx:latest
+      ports:
+        - containerPort: 80

test/conformance/kuttl/background-only/policy/no-admission-report/01-policy.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- policy.yaml
+assert:
+- policy-assert.yaml

test/conformance/kuttl/background-only/policy/no-admission-report/02-resource.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- resource.yaml

test/conformance/kuttl/background-only/policy/no-admission-report/03-report.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+error:
+- admission-report.yaml

test/conformance/kuttl/background-only/policy/no-admission-report/README.md

@@ -0,0 +1,9 @@
+## Description
+
+This test creates a policy with `admission` set to `false`.
+Then it creates a resource that violates the policy.
+
+## Expected Behavior
+
+The resource creates fine as the policy doesn't apply at admission time.
+No admission report is created.

test/conformance/kuttl/background-only/policy/no-admission-report/admission-report.yaml

@@ -0,0 +1,7 @@
+apiVersion: kyverno.io/v1alpha2
+kind: AdmissionReport
+metadata:
+  ownerReferences:
+  - apiVersion: v1
+    kind: Pod
+    name: pod

test/conformance/kuttl/background-only/policy/no-admission-report/policy-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: validate
+spec: {}
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/kuttl/background-only/policy/no-admission-report/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: validate
+spec:
+  validationFailureAction: Enforce
+  admission: false
+  background: true
+  rules:
+  - name: validate
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      deny: {}

test/conformance/kuttl/background-only/policy/no-admission-report/resource.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod
+spec:
+  containers:
+    - name: container
+      image: nginx:latest
+      ports:
+        - containerPort: 80

test/conformance/kuttl/background-only/policy/not-rejected/01-policy.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- policy.yaml
+assert:
+- policy-assert.yaml

test/conformance/kuttl/background-only/policy/not-rejected/02-resource.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- resource.yaml

test/conformance/kuttl/background-only/policy/not-rejected/README.md

@@ -0,0 +1,8 @@
+## Description
+
+This test creates a policy with `admission` set to `false`.
+Then it creates a resource that violates the policy.
+
+## Expected Behavior
+
+The resource creates fine as the policy doesn't apply at admission time.

test/conformance/kuttl/background-only/policy/not-rejected/policy-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: validate
+spec: {}
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/kuttl/background-only/policy/not-rejected/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: validate
+spec:
+  validationFailureAction: Enforce
+  admission: false
+  background: true
+  rules:
+  - name: validate
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      deny: {}

test/conformance/kuttl/background-only/policy/not-rejected/resource.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod
+spec:
+  containers:
+    - name: container
+      image: nginx:latest
+      ports:
+        - containerPort: 80

test/conformance/kuttl/policy-validation/policy/admission-disabled/01-policy.yaml

@@ -0,0 +1,8 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- file: policy-validate.yaml
+- file: policy-mutate.yaml
+  shouldFail: true
+- file: policy-verify-image.yaml
+  shouldFail: true

test/conformance/kuttl/policy-validation/policy/admission-disabled/README.md

@@ -0,0 +1,7 @@
+## Description
+
+This test tries to create various policies with `admission` set to `false`.
+
+## Expected Behavior
+
+Policies containing mutation, image verification or generation rules should be rejected.

test/conformance/kuttl/policy-validation/policy/admission-disabled/policy-mutate.yaml

@@ -0,0 +1,22 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: mutate
+spec:
+  validationFailureAction: Audit
+  admission: false
+  background: true
+  rules:
+  - name: mutate
+    match:
+      resources:
+        kinds:
+        - Pod
+        - Service
+        - ConfigMap
+        - Secret
+    mutate:
+      patchStrategicMerge:
+        metadata:
+          labels:
+            foo: bar

test/conformance/kuttl/policy-validation/policy/admission-disabled/policy-validate.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: validate
+spec:
+  validationFailureAction: Audit
+  admission: false
+  background: true
+  rules:
+  - name: validate
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      deny: {}

test/conformance/kuttl/policy-validation/policy/admission-disabled/policy-verify-image.yaml

@@ -0,0 +1,26 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: verify-image
+spec:
+  validationFailureAction: Audit
+  admission: false
+  background: true
+  rules:
+  - name: verify-image
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    verifyImages:
+    - imageReferences:
+      - "ghcr.io/kyverno/test-verify-image:*"
+      attestors:
+      - entries:
+        - keys:
+            publicKeys: |-
+              -----BEGIN PUBLIC KEY-----
+              MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE8nXRh950IZbRj8Ra/N9sbqOPZrfM
+              5/KAQN0/KjHcorm/J5yctVd7iEcnessRQjU917hmKO6JWVGHpDguIyakZA==
+              -----END PUBLIC KEY-----

test/conformance/kuttl/policy-validation/policy/all-disabled/01-policy.yaml

@@ -0,0 +1,5 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- file: policy.yaml
+  shouldFail: true

test/conformance/kuttl/policy-validation/policy/all-disabled/README.md

@@ -0,0 +1,7 @@
+## Description
+
+This test tries to create a policy with both `admission` and `background` set to `false`.
+
+## Expected Behavior
+
+Policy should be rejected.

test/conformance/kuttl/policy-validation/policy/all-disabled/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: all-disabled
+spec:
+  validationFailureAction: Audit
+  admission: false
+  background: false
+  rules:
+  - name: validate
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      deny: {}