feat: add chainsaw tests for exceptions (#9672)

* feat: add chainsaw tests for exceptions

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>

* fix test

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>

---------

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>
Co-authored-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

test/conformance/chainsaw/reports/background/exception-with-conditions/README.md

@@ -0,0 +1,12 @@
+## Description
+
+This test creates a policy, a policy exception and a pod.
+It makes sure the generated background scan report contains a skipped result instead of a failed one.
+
+## Steps
+
+1.  - Create a pod named `apps`
+2.  - Create a cluster policy
+    - Assert the policy becomes ready
+3.  - Create a policy exception for the cluster policy created above, configured to apply to pod named `apps`
+4.  - Assert that a policy report exists with a skipped result

test/conformance/chainsaw/reports/background/exception-with-conditions/chainsaw-test.yaml

@@ -0,0 +1,31 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: exception-with-conditions
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: ns.yaml
+    - assert:
+        file: ns.yaml
+  - name: step-02
+    try:
+    - apply:
+        file: deployment.yaml
+  - name: step-03
+    try:
+    - apply:
+        file: policy.yaml
+    - assert:
+        file: policy-assert.yaml
+  - name: step-04
+    try:
+    - apply:
+        file: exception.yaml
+  - name: step-05
+    try:
+    - assert:
+        file: report-assert.yaml

test/conformance/chainsaw/reports/background/exception-with-conditions/deployment.yaml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: new-deployment
+  namespace: dev-ns
+  labels:
+    env: dev
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: new-deployment
+  template:
+    metadata:
+      labels:
+        app: new-deployment
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.14.2
+        ports:
+        - containerPort: 80

test/conformance/chainsaw/reports/background/exception-with-conditions/exception.yaml

@@ -0,0 +1,21 @@
+apiVersion: kyverno.io/v2beta1
+kind: PolicyException
+metadata:
+  name: container-exception
+spec:
+  exceptions:
+  - policyName: check-deployment-replicas
+    ruleNames:
+    - check-deployment-replicas
+  match:
+    any:
+    - resources:
+        kinds:
+        - Deployment
+        namespaces:
+        - dev-ns
+  conditions:
+    any:
+    - key: "{{ request.object.metadata.labels.env || '' }}"
+      operator: Equals
+      value: dev

test/conformance/chainsaw/reports/background/exception-with-conditions/ns.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: dev-ns

test/conformance/chainsaw/reports/background/exception-with-conditions/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: check-deployment-replicas
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/reports/background/exception-with-conditions/policy.yaml

@@ -0,0 +1,22 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: check-deployment-replicas
+spec:
+  validationFailureAction: Enforce
+  background: true
+  rules:
+  - name: check-deployment-replicas
+    match:
+      any:
+      - resources:
+          kinds:
+          - Deployment
+    validate:
+      message: "Deployment should have at most 1 replica"
+      deny:
+        conditions:
+          any:
+          - key: "{{request.object.spec.replicas}}"
+            operator: GreaterThan
+            value: "2"

test/conformance/chainsaw/reports/background/exception-with-conditions/report-assert.yaml

@@ -0,0 +1,27 @@
+apiVersion: wgpolicyk8s.io/v1alpha2
+kind: PolicyReport
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kyverno
+  namespace: dev-ns
+  ownerReferences:
+  - apiVersion: apps/v1
+    kind: Deployment
+    name: new-deployment
+results:
+- policy: check-deployment-replicas
+  result: skip
+  rule: check-deployment-replicas
+  scored: true
+  source: kyverno
+scope:
+  apiVersion: apps/v1
+  kind: Deployment
+  name: new-deployment
+  namespace: dev-ns
+summary:
+  error: 0
+  fail: 0
+  pass: 0
+  skip: 1
+  warn: 0

test/conformance/chainsaw/reports/background/exception-with-podsecurity/README.md

@@ -0,0 +1,12 @@
+## Description
+
+This test creates a policy, a policy exception and a pod.
+It makes sure the generated background scan report contains a skipped result instead of a failed one.
+
+## Steps
+
+1.  - Create a pod named `nginx`
+2.  - Create a cluster policy
+    - Assert the policy becomes ready
+3.  - Create a policy exception for the cluster policy created above, configured to apply to pod named `nginx`
+4.  - Assert that a policy report exists with a skipped result

test/conformance/chainsaw/reports/background/exception-with-podsecurity/chainsaw-test.yaml

@@ -0,0 +1,25 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: exception-with-podsecurity
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: pod.yaml
+  - name: step-02
+    try:
+    - apply:
+        file: policy.yaml
+    - assert:
+        file: policy-assert.yaml
+  - name: step-03
+    try:
+    - apply:
+        file: exception.yaml
+  - name: step-04
+    try:
+    - assert:
+        file: report-assert.yaml

test/conformance/chainsaw/reports/background/exception-with-podsecurity/exception.yaml

@@ -0,0 +1,19 @@
+apiVersion: kyverno.io/v2beta1
+kind: PolicyException
+metadata:
+  name: pod-security-exception
+spec:
+  exceptions:
+  - policyName: psa-1
+    ruleNames:
+    - restricted
+  match:
+    any:
+    - resources:
+        kinds:
+        - Pod
+  podSecurity:
+    - controlName: Capabilities
+      images:
+          - nginx*
+          - redis*

test/conformance/chainsaw/reports/background/exception-with-podsecurity/pod.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx
+  namespace: default
+spec:
+  containers:
+  - name: container01
+    image: nginx:1.1.9
+    securityContext:
+      allowPrivilegeEscalation: false
+      runAsNonRoot: true
+      seccompProfile:
+        type: RuntimeDefault
+      capabilities:
+        add:
+        - SYS_ADMIN
+        drop:
+        - ALL

test/conformance/chainsaw/reports/background/exception-with-podsecurity/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: psa-1
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/reports/background/exception-with-podsecurity/policy.yaml

@@ -0,0 +1,20 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: psa-1
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+spec:
+  background: true
+  validationFailureAction: Enforce
+  rules:
+  - name: restricted
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    validate:
+      podSecurity:
+        level: restricted
+        version: latest

test/conformance/chainsaw/reports/background/exception-with-podsecurity/report-assert.yaml

@@ -0,0 +1,26 @@
+apiVersion: wgpolicyk8s.io/v1alpha2
+kind: PolicyReport
+metadata:
+  namespace: default
+  ownerReferences:
+  - apiVersion: v1
+    kind: Pod
+    name: nginx
+results:
+- policy: psa-1
+  properties:
+    exception: pod-security-exception
+  result: skip
+  rule: restricted
+  scored: true
+  source: kyverno
+scope:
+  apiVersion: v1
+  kind: Pod
+  name: nginx
+summary:
+  error: 0
+  fail: 0
+  pass: 0
+  skip: 1
+  warn: 0