chore: run background-only tests with chainsaw (#8943)

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>

.github/workflows/conformance.yaml

@@ -196,6 +196,7 @@ jobs:
           # - verify-manifests
           # - verifyImages
           - webhooks
+          - background-only
     needs: prepare-images
     name: chainsaw - ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
     steps:

test/conformance/chainsaw/_config/common.yaml

@@ -3,6 +3,9 @@ kind: Configuration
 metadata:
   name: congiguration
 spec:
+  timeouts:
+    assert: 90s
+    error: 90s
   parallel: 1
   fullName: true
   failFast: true

test/conformance/chainsaw/background-only/cluster-policy/no-admission-event/01-policy.yaml

@@ -0,0 +1,10 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: policy
+spec:
+  try:
+  - apply:
+      file: policy.yaml
+  - assert:
+      file: policy-assert.yaml

test/conformance/chainsaw/background-only/cluster-policy/no-admission-event/02-resource.yaml

@@ -0,0 +1,8 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: resource
+spec:
+  try:
+  - apply:
+      file: resource.yaml

test/conformance/chainsaw/background-only/cluster-policy/no-admission-event/03-event.yaml

@@ -0,0 +1,12 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: event
+spec:
+  try:
+  - assert:
+      file: background-event.yaml
+  - error:
+      file: admission-event.yaml
+  catch:
+  - events: {}

test/conformance/chainsaw/background-only/cluster-policy/no-admission-event/README.md

@@ -0,0 +1,10 @@
+## Description
+
+This test creates a policy with `admission` set to `false`.
+Then it creates a resource that violates the policy.
+
+## Expected Behavior
+
+The resource creates fine as the policy doesn't apply at admission time.
+No admission ezvent is created.
+One background event is created.

test/conformance/chainsaw/background-only/cluster-policy/no-admission-event/admission-event.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+involvedObject:
+  apiVersion: v1
+  kind: Pod
+  name: pod
+kind: Event
+metadata: {}
+reportingComponent: kyverno-admission

test/conformance/chainsaw/background-only/cluster-policy/no-admission-event/background-event.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+involvedObject:
+  apiVersion: v1
+  kind: Pod
+  name: pod
+kind: Event
+metadata: {}
+reportingComponent: kyverno-scan

test/conformance/chainsaw/background-only/cluster-policy/no-admission-event/policy-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: validate
+spec: {}
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/background-only/cluster-policy/no-admission-event/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: validate
+spec:
+  validationFailureAction: Enforce
+  admission: false
+  background: true
+  rules:
+  - name: validate
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      deny: {}

test/conformance/chainsaw/background-only/cluster-policy/no-admission-event/resource.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod
+spec:
+  containers:
+    - name: container
+      image: nginx:latest
+      ports:
+        - containerPort: 80

test/conformance/chainsaw/background-only/cluster-policy/no-admission-report/01-policy.yaml

@@ -0,0 +1,10 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: policy
+spec:
+  try:
+  - apply:
+      file: policy.yaml
+  - assert:
+      file: policy-assert.yaml

test/conformance/chainsaw/background-only/cluster-policy/no-admission-report/02-resource.yaml

@@ -0,0 +1,8 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: resource
+spec:
+  try:
+  - apply:
+      file: resource.yaml

test/conformance/chainsaw/background-only/cluster-policy/no-admission-report/03-report.yaml

@@ -0,0 +1,8 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: report
+spec:
+  try:
+  - error:
+      file: admission-report.yaml

test/conformance/chainsaw/background-only/cluster-policy/no-admission-report/README.md

@@ -0,0 +1,9 @@
+## Description
+
+This test creates a policy with `admission` set to `false`.
+Then it creates a resource that violates the policy.
+
+## Expected Behavior
+
+The resource creates fine as the policy doesn't apply at admission time.
+No admission report is created.

test/conformance/chainsaw/background-only/cluster-policy/no-admission-report/admission-report.yaml

@@ -0,0 +1,7 @@
+apiVersion: kyverno.io/v1alpha2
+kind: AdmissionReport
+metadata:
+  ownerReferences:
+  - apiVersion: v1
+    kind: Pod
+    name: pod

test/conformance/chainsaw/background-only/cluster-policy/no-admission-report/policy-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: validate
+spec: {}
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/background-only/cluster-policy/no-admission-report/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: validate
+spec:
+  validationFailureAction: Enforce
+  admission: false
+  background: true
+  rules:
+  - name: validate
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      deny: {}

test/conformance/chainsaw/background-only/cluster-policy/no-admission-report/resource.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod
+spec:
+  containers:
+    - name: container
+      image: nginx:latest
+      ports:
+        - containerPort: 80

test/conformance/chainsaw/background-only/cluster-policy/not-rejected/01-policy.yaml

@@ -0,0 +1,10 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: policy
+spec:
+  try:
+  - apply:
+      file: policy.yaml
+  - assert:
+      file: policy-assert.yaml

test/conformance/chainsaw/background-only/cluster-policy/not-rejected/02-resource.yaml

@@ -0,0 +1,8 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: resource
+spec:
+  try:
+  - apply:
+      file: resource.yaml

test/conformance/chainsaw/background-only/cluster-policy/not-rejected/README.md

@@ -0,0 +1,8 @@
+## Description
+
+This test creates a policy with `admission` set to `false`.
+Then it creates a resource that violates the policy.
+
+## Expected Behavior
+
+The resource creates fine as the policy doesn't apply at admission time.

test/conformance/chainsaw/background-only/cluster-policy/not-rejected/policy-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: validate
+spec: {}
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/background-only/cluster-policy/not-rejected/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: validate
+spec:
+  validationFailureAction: Enforce
+  admission: false
+  background: true
+  rules:
+  - name: validate
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      deny: {}

test/conformance/chainsaw/background-only/cluster-policy/not-rejected/resource.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod
+spec:
+  containers:
+    - name: container
+      image: nginx:latest
+      ports:
+        - containerPort: 80

test/conformance/chainsaw/background-only/policy/no-admission-event/01-policy.yaml

@@ -0,0 +1,10 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: policy
+spec:
+  try:
+  - apply:
+      file: policy.yaml
+  - assert:
+      file: policy-assert.yaml

test/conformance/chainsaw/background-only/policy/no-admission-event/02-resource.yaml

@@ -0,0 +1,8 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: resource
+spec:
+  try:
+  - apply:
+      file: resource.yaml

test/conformance/chainsaw/background-only/policy/no-admission-event/03-event.yaml

@@ -0,0 +1,10 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: event
+spec:
+  try:
+  - assert:
+      file: background-event.yaml
+  - error:
+      file: admission-event.yaml

test/conformance/chainsaw/background-only/policy/no-admission-event/README.md

@@ -0,0 +1,10 @@
+## Description
+
+This test creates a policy with `admission` set to `false`.
+Then it creates a resource that violates the policy.
+
+## Expected Behavior
+
+The resource creates fine as the policy doesn't apply at admission time.
+No admission ezvent is created.
+One background event is created.

test/conformance/chainsaw/background-only/policy/no-admission-event/admission-event.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+involvedObject:
+  apiVersion: v1
+  kind: Pod
+  name: pod
+kind: Event
+metadata: {}
+reportingComponent: kyverno-admission

test/conformance/chainsaw/background-only/policy/no-admission-event/background-event.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+involvedObject:
+  apiVersion: v1
+  kind: Pod
+  name: pod
+kind: Event
+metadata: {}
+reportingComponent: kyverno-scan

test/conformance/chainsaw/background-only/policy/no-admission-event/policy-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: validate
+spec: {}
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/background-only/policy/no-admission-event/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: validate
+spec:
+  validationFailureAction: Enforce
+  admission: false
+  background: true
+  rules:
+  - name: validate
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      deny: {}

test/conformance/chainsaw/background-only/policy/no-admission-event/resource.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod
+spec:
+  containers:
+    - name: container
+      image: nginx:latest
+      ports:
+        - containerPort: 80

test/conformance/chainsaw/background-only/policy/no-admission-report/01-policy.yaml

@@ -0,0 +1,10 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: policy
+spec:
+  try:
+  - apply:
+      file: policy.yaml
+  - assert:
+      file: policy-assert.yaml

test/conformance/chainsaw/background-only/policy/no-admission-report/02-resource.yaml

@@ -0,0 +1,8 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: resource
+spec:
+  try:
+  - apply:
+      file: resource.yaml

test/conformance/chainsaw/background-only/policy/no-admission-report/03-report.yaml

@@ -0,0 +1,8 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: report
+spec:
+  try:
+  - error:
+      file: admission-report.yaml

test/conformance/chainsaw/background-only/policy/no-admission-report/README.md

@@ -0,0 +1,9 @@
+## Description
+
+This test creates a policy with `admission` set to `false`.
+Then it creates a resource that violates the policy.
+
+## Expected Behavior
+
+The resource creates fine as the policy doesn't apply at admission time.
+No admission report is created.

test/conformance/chainsaw/background-only/policy/no-admission-report/admission-report.yaml

@@ -0,0 +1,7 @@
+apiVersion: kyverno.io/v1alpha2
+kind: AdmissionReport
+metadata:
+  ownerReferences:
+  - apiVersion: v1
+    kind: Pod
+    name: pod

test/conformance/chainsaw/background-only/policy/no-admission-report/policy-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: validate
+spec: {}
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/background-only/policy/no-admission-report/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: validate
+spec:
+  validationFailureAction: Enforce
+  admission: false
+  background: true
+  rules:
+  - name: validate
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      deny: {}

test/conformance/chainsaw/background-only/policy/no-admission-report/resource.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod
+spec:
+  containers:
+    - name: container
+      image: nginx:latest
+      ports:
+        - containerPort: 80

test/conformance/chainsaw/background-only/policy/not-rejected/01-policy.yaml

@@ -0,0 +1,10 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: policy
+spec:
+  try:
+  - apply:
+      file: policy.yaml
+  - assert:
+      file: policy-assert.yaml

test/conformance/chainsaw/background-only/policy/not-rejected/02-resource.yaml

@@ -0,0 +1,9 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: resource
+spec:
+  try:
+  - apply:
+      file: resource.yaml

test/conformance/chainsaw/background-only/policy/not-rejected/README.md

@@ -0,0 +1,8 @@
+## Description
+
+This test creates a policy with `admission` set to `false`.
+Then it creates a resource that violates the policy.
+
+## Expected Behavior
+
+The resource creates fine as the policy doesn't apply at admission time.

test/conformance/chainsaw/background-only/policy/not-rejected/policy-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: validate
+spec: {}
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/background-only/policy/not-rejected/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: validate
+spec:
+  validationFailureAction: Enforce
+  admission: false
+  background: true
+  rules:
+  - name: validate
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      deny: {}

test/conformance/chainsaw/background-only/policy/not-rejected/resource.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod
+spec:
+  containers:
+    - name: container
+      image: nginx:latest
+      ports:
+        - containerPort: 80