chore: migrate tests to chainsaw (#8997)

* chore: migrate tests to chainsaw

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* cleanup

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* cleanup

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix: exec timeout

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* exceptions

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* exceptions

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* filter

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

.github/workflows/conformance.yaml

@@ -179,11 +179,11 @@ jobs:
         tests:
           - autogen
           - background-only
-          # - cleanup
+          - cleanup
-          # - deferred
+          - deferred
           - events
-          # - exceptions
+          - exceptions
-          # - filter
+          - filter
           # - generate/clusterpolicy
           # - generate/policy
           # - generate/validation

test/conformance/chainsaw/_config/common.yaml

@@ -6,6 +6,7 @@ spec:
   timeouts:
     assert: 90s
     error: 90s
+    exec: 90s
   parallel: 1
   fullName: true
   failFast: true

test/conformance/chainsaw/cleanup/clusterpolicy/cleanup-pod/01-rbac.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: rbac
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: rbac.yaml

test/conformance/chainsaw/cleanup/clusterpolicy/cleanup-pod/02-pod.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: pod
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: pod.yaml
+  - assert:
+      file: pod-assert.yaml

test/conformance/chainsaw/cleanup/clusterpolicy/cleanup-pod/03-policy.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: policy
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: policy.yaml
+  - assert:
+      file: policy.yaml

test/conformance/chainsaw/cleanup/clusterpolicy/cleanup-pod/04-sleep.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: sleep
+spec:
+  timeouts: {}
+  try:
+  - command:
+      args:
+      - "65"
+      check: null
+      entrypoint: sleep

test/conformance/chainsaw/cleanup/clusterpolicy/cleanup-pod/05-check.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: check
+spec:
+  timeouts: {}
+  try:
+  - error:
+      file: pod-assert.yaml

test/conformance/chainsaw/cleanup/clusterpolicy/cleanup-pod/README.md

@@ -0,0 +1,9 @@
+# ## Description
+
+This test cleans up pods via a cluster cleanup policy.
+
+## Expected Behavior
+
+The pod `default/example` is cleaned up successfully.
+
+## Reference Issue(s)

test/conformance/chainsaw/cleanup/clusterpolicy/cleanup-pod/pod-assert.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: example
+  namespace: default

test/conformance/chainsaw/cleanup/clusterpolicy/cleanup-pod/pod.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: example
+  namespace: default
+spec:
+  containers:
+  - image: nginx:latest
+    name: example

test/conformance/chainsaw/cleanup/clusterpolicy/cleanup-pod/policy.yaml

@@ -0,0 +1,20 @@
+apiVersion: kyverno.io/v2beta1
+kind: ClusterCleanupPolicy
+metadata:
+  name: cleanup-pod
+spec:
+  match:
+    any:
+      - resources:
+          kinds:
+            - Pod
+  conditions:
+    all:
+      - key: "{{ target.metadata.name }}"
+        operator: Equals
+        value: example
+      - key: "{{ target.metadata.namespace }}"
+        operator: Equals
+        value: default
+  ## execute every minute
+  schedule: "*/1 * * * *"

test/conformance/chainsaw/cleanup/clusterpolicy/cleanup-pod/rbac.yaml

@@ -0,0 +1,26 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: test-cleanup-pod
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - delete
+  - list
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: test-cleanup-pod
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: test-cleanup-pod
+subjects:
+- kind: ServiceAccount
+  name: kyverno-cleanup-controller
+  namespace: kyverno

test/conformance/chainsaw/cleanup/clusterpolicy/context-cleanup-pod/01-rbac.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: rbac
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: rbac.yaml

test/conformance/chainsaw/cleanup/clusterpolicy/context-cleanup-pod/02-pod.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: pod
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: pod.yaml
+  - assert:
+      file: pod-assert.yaml

test/conformance/chainsaw/cleanup/clusterpolicy/context-cleanup-pod/03-policy.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: policy
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: policy.yaml
+  - assert:
+      file: policy.yaml

test/conformance/chainsaw/cleanup/clusterpolicy/context-cleanup-pod/04-sleep.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: sleep
+spec:
+  timeouts: {}
+  try:
+  - command:
+      args:
+      - "5"
+      check: null
+      entrypoint: sleep

test/conformance/chainsaw/cleanup/clusterpolicy/context-cleanup-pod/05-check.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: check
+spec:
+  timeouts: {}
+  try:
+  - error:
+      file: pod-assert.yaml

test/conformance/chainsaw/cleanup/clusterpolicy/context-cleanup-pod/README.md

@@ -0,0 +1,9 @@
+# ## Description
+
+This test cleans up pods via a cluster cleanup policy.
+
+## Expected Behavior
+
+The pod `{{ varname }}` in the namespace `{{ varNamespace }}` set by context variable is cleaned up successfully.
+
+## Reference Issue(s)

test/conformance/chainsaw/cleanup/clusterpolicy/context-cleanup-pod/pod-assert.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: example
+  namespace: default

test/conformance/chainsaw/cleanup/clusterpolicy/context-cleanup-pod/pod.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: example
+  namespace: default
+spec:
+  containers:
+  - image: nginx:latest
+    name: example

test/conformance/chainsaw/cleanup/clusterpolicy/context-cleanup-pod/policy.yaml

@@ -0,0 +1,28 @@
+apiVersion: kyverno.io/v2beta1
+kind: ClusterCleanupPolicy
+metadata:
+  name: cleanup-pod
+spec:
+  context:
+    - name: varNamespace
+      apiCall:
+        urlPath: "/api/v1/namespaces/default"
+        jmesPath: metadata.name
+    - name: varname
+      variable:
+        value: "example"
+  match:
+    any:
+      - resources:
+          kinds:
+            - Pod
+  conditions:
+    all:
+      - key: "{{ target.metadata.name }}"
+        operator: Equals
+        value: "{{ varname }}"
+      - key: "{{ target.metadata.namespace }}"
+        operator: Equals
+        value: "{{ varNamespace }}"
+  ## execute every minute
+  schedule: "*/1 * * * *"

test/conformance/chainsaw/cleanup/clusterpolicy/context-cleanup-pod/rbac.yaml

@@ -0,0 +1,26 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: test-cleanup-pod
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - delete
+  - list
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: test-cleanup-pod
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: test-cleanup-pod
+subjects:
+- kind: ServiceAccount
+  name: kyverno-cleanup-controller
+  namespace: kyverno

test/conformance/chainsaw/cleanup/policy/cleanup-pod/01-rbac.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: rbac
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: rbac.yaml

test/conformance/chainsaw/cleanup/policy/cleanup-pod/02-pod.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: pod
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: pod.yaml
+  - assert:
+      file: pod-assert.yaml

test/conformance/chainsaw/cleanup/policy/cleanup-pod/03-policy.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: policy
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: policy.yaml
+  - assert:
+      file: policy.yaml

test/conformance/chainsaw/cleanup/policy/cleanup-pod/04-sleep.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: sleep
+spec:
+  timeouts: {}
+  try:
+  - command:
+      args:
+      - "65"
+      check: null
+      entrypoint: sleep

test/conformance/chainsaw/cleanup/policy/cleanup-pod/05-check.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: check
+spec:
+  timeouts: {}
+  try:
+  - error:
+      file: pod-assert.yaml

test/conformance/chainsaw/cleanup/policy/cleanup-pod/README.md

@@ -0,0 +1,10 @@
+# ## Description
+
+This test cleans up pods via a namespaced cleanup policy.
+
+## Expected Behavior
+
+The pod `default/example` is cleaned up successfully.
+
+
+## Reference Issue(s)

test/conformance/chainsaw/cleanup/policy/cleanup-pod/pod-assert.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: example
+  namespace: default

test/conformance/chainsaw/cleanup/policy/cleanup-pod/pod.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: example
+  namespace: default
+spec:
+  containers:
+  - image: nginx:latest
+    name: example

test/conformance/chainsaw/cleanup/policy/cleanup-pod/policy.yaml

@@ -0,0 +1,18 @@
+apiVersion: kyverno.io/v2beta1
+kind: CleanupPolicy
+metadata:
+  name: cleanup-pod
+  namespace: default
+spec:
+  match:
+    any:
+    - resources:
+        kinds:
+          - Pod
+  conditions:
+    any:
+    - key: "{{ target.metadata.name }}"
+      operator: Equals
+      value: example
+  ## execute every minute
+  schedule: "*/1 * * * *"

test/conformance/chainsaw/cleanup/policy/cleanup-pod/rbac.yaml

@@ -0,0 +1,26 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: test-cleanup-pod
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - delete
+  - list
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: test-cleanup-pod
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: test-cleanup-pod
+subjects:
+- kind: ServiceAccount
+  name: kyverno-cleanup-controller
+  namespace: kyverno

test/conformance/chainsaw/cleanup/validation/cron-format/01-policy.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: policy
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: policy.yaml
+  - assert:
+      file: policy.yaml

test/conformance/chainsaw/cleanup/validation/cron-format/02-clusterpolicy.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: clusterpolicy
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: clusterpolicy.yaml
+  - assert:
+      file: clusterpolicy.yaml

test/conformance/chainsaw/cleanup/validation/cron-format/03-invalidpolicy.yaml

@@ -0,0 +1,10 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: invalidpolicy
+spec:
+  try:
+  - apply:
+      file: invalidpolicy.yaml
+      check:
+        (error == null): false

test/conformance/chainsaw/cleanup/validation/cron-format/README.md

@@ -0,0 +1,4 @@
+## Description
+
+This test creates tries to create two cleanup policies, one with a valid schedule and one with an invalid schedule.
+The creation of the one with the valid schedule is expected to succeed while the one with the invalid schedule is expected to fail.

test/conformance/chainsaw/cleanup/validation/cron-format/clusterpolicy.yaml

@@ -0,0 +1,16 @@
+apiVersion: kyverno.io/v2beta1
+kind: ClusterCleanupPolicy
+metadata:
+  name: cleanuppolicy
+spec:
+  match:
+    any:
+    - resources:
+        kinds:
+          - Pod
+  conditions:
+    any:
+    - key: "{{ target.metadata.name }}"
+      operator: Equals
+      value: example
+  schedule: "*/2 * * * *"

test/conformance/chainsaw/cleanup/validation/cron-format/invalidpolicy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v2beta1
+kind: CleanupPolicy
+metadata:
+  name: cleanuppolicy
+  namespace: default
+spec:
+  match:
+    any:
+    - resources:
+        kinds:
+          - Pod
+  conditions:
+    any:
+    - key: "{{ target.metadata.name }}"
+      operator: Equals
+      value: example
+  schedule: "invalid-schedule"

test/conformance/chainsaw/cleanup/validation/cron-format/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v2beta1
+kind: CleanupPolicy
+metadata:
+  name: cleanuppolicy
+  namespace: default
+spec:
+  match:
+    any:
+    - resources:
+        kinds:
+          - Pod
+  conditions:
+    any:
+    - key: "{{ target.metadata.name }}"
+      operator: Equals
+      value: example
+  schedule: "*/2 * * * *"

test/conformance/chainsaw/cleanup/validation/no-user-info-in-match/01-cleanuppolicy.yaml

@@ -0,0 +1,18 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: cleanuppolicy
+spec:
+  try:
+  - apply:
+      file: cleanuppolicy-with-subjects.yaml
+      check:
+        (error == null): false
+  - apply:
+      file: cleanuppolicy-with-roles.yaml
+      check:
+        (error == null): false
+  - apply:
+      file: cleanuppolicy-with-clusterroles.yaml
+      check:
+        (error == null): false

test/conformance/chainsaw/cleanup/validation/no-user-info-in-match/README.md

@@ -0,0 +1,8 @@
+## Description
+
+This test creates a cleanup policy containing user infos in `match` statement.
+The creation should fail as cleanup policies with user infos are not allowed.
+
+## Steps
+
+1.  - Try create a couple of cleanup policies, expecting the creation to fail because they contain user infos

test/conformance/chainsaw/cleanup/validation/no-user-info-in-match/cleanuppolicy-with-clusterroles.yaml

@@ -0,0 +1,13 @@
+apiVersion: kyverno.io/v2beta1
+kind: ClusterCleanupPolicy
+metadata:
+  name: cleanuppolicy
+spec:
+  match:
+    any:
+      - resources:
+          kinds:
+            - Pod
+        clusterRoles:
+          - clusteradmin
+  schedule: '* * * * *'

test/conformance/chainsaw/cleanup/validation/no-user-info-in-match/cleanuppolicy-with-roles.yaml

@@ -0,0 +1,13 @@
+apiVersion: kyverno.io/v2beta1
+kind: ClusterCleanupPolicy
+metadata:
+  name: cleanuppolicy
+spec:
+  match:
+    any:
+      - resources:
+          kinds:
+            - Pod
+        roles:
+          - admin
+  schedule: '* * * * *'

test/conformance/chainsaw/cleanup/validation/no-user-info-in-match/cleanuppolicy-with-subjects.yaml

@@ -0,0 +1,14 @@
+apiVersion: kyverno.io/v2beta1
+kind: ClusterCleanupPolicy
+metadata:
+  name: cleanuppolicy
+spec:
+  match:
+    any:
+      - resources:
+          kinds:
+            - Pod
+        subjects:
+          - kind: User
+            name: chip
+  schedule: '* * * * *'

test/conformance/chainsaw/cleanup/validation/not-supported-attributes-in-context/01-cleanup-policy.yaml

@@ -0,0 +1,14 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: cleanup-policy
+spec:
+  try:
+  - apply:
+      file: cleanuppolicy-with-image-registry.yaml
+      check:
+        (error == null): false
+  - apply:
+      file: cleanuppolicy-with-configmap.yaml
+      check:
+        (error == null): false

test/conformance/chainsaw/cleanup/validation/not-supported-attributes-in-context/cleanuppolicy-with-configmap.yaml

@@ -0,0 +1,25 @@
+apiVersion: kyverno.io/v2beta1
+kind: ClusterCleanupPolicy
+metadata:
+  name: cleanup-pod
+spec:
+  context:
+    - name: configData
+      configMap:
+        name: some-config-map
+        namespace: default
+  match:
+    any:
+      - resources:
+          kinds:
+            - Pod
+  conditions:
+    all:
+      - key: "{{ target.metadata.name }}"
+        operator: Equals
+        value: example
+      - key: "{{ target.metadata.namespace }}"
+        operator: Equals
+        value: default
+  ## execute every minute
+  schedule: "*/1 * * * *"

test/conformance/chainsaw/cleanup/validation/not-supported-attributes-in-context/cleanuppolicy-with-image-registry.yaml

@@ -0,0 +1,24 @@
+apiVersion: kyverno.io/v2beta1
+kind: ClusterCleanupPolicy
+metadata:
+  name: cleanup-pod
+spec:
+  context:
+    - name: imageData
+      imageRegistry:
+        reference: "ghcr.io/kyverno/kyverno"
+  match:
+    any:
+      - resources:
+          kinds:
+            - Pod
+  conditions:
+    all:
+      - key: "{{ target.metadata.name }}"
+        operator: Equals
+        value: "example"
+      - key: "{{ target.metadata.namespace }}"
+        operator: Equals
+        value: default
+  ## execute every minute
+  schedule: "*/1 * * * *"

test/conformance/chainsaw/deferred/dependencies/01-apply-manifests.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: apply-manifests
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: manifests.yaml
+  - assert:
+      file: policy-assert.yaml

test/conformance/chainsaw/deferred/dependencies/02-testcase.yaml

@@ -0,0 +1,10 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: testcase
+spec:
+  try:
+  - apply:
+      file: deploy.yaml
+      check:
+        (error == null): false

test/conformance/chainsaw/deferred/dependencies/README.md

@@ -0,0 +1,12 @@
+## Description
+
+This test checks for handling of variable dependencies with deferred lookups
+
+## Expected Behavior
+
+The deployment should fail
+
+## Reference Issues
+
+https://github.com/kyverno/kyverno/issues/7486
+

test/conformance/chainsaw/deferred/dependencies/deploy.yaml

@@ -0,0 +1,28 @@
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test
+  namespace: acme-fitness
+  labels:
+    app: kubecost-cost-analyzer
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: kubecost-cost-analyzer
+  template:
+    metadata:
+      labels:
+        app: kubecost-cost-analyzer
+    spec:
+      containers:
+      - name: cost-model
+        image: nginx:1.14.2
+        resources:
+          requests:
+            cpu: 350m
+            memory: 500Mi
+          limits:
+            memory: 2Gi

test/conformance/chainsaw/deferred/dependencies/manifests.yaml

@@ -0,0 +1,73 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: acme-fitness
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: enforce-company-budget
+spec:
+  validationFailureAction: Enforce
+  rules:
+  - name: check-kubecost-budget
+    match:
+      any:
+      - resources:
+          kinds:
+          - Deployment
+          operations:
+          - CREATE
+    context:
+    # Mocked response from the Kubecost prediction API until it natively supports JSON input. 
+    # Get the predicted amount of the Deployment and transform to get the totalMonthlyRate.
+    - name: predictedcost
+      variable:
+        jmesPath: '[0].costChange.totalMonthlyRate'
+        value:
+          - namespace: acme-fitness
+            controllerKind: deployment
+            controllerName: test
+            costBefore:
+              totalMonthlyRate: 0
+              cpuMonthlyRate: 0
+              ramMonthlyRate: 0
+              gpuMonthlyRate: 0
+              monthlyCPUCoreHours: 0
+              monthlyRAMByteHours: 0
+              monthlyGPUHours: 0
+            costAfter:
+              totalMonthlyRate: 28.839483652409793
+              cpuMonthlyRate: 24.295976357646456
+              ramMonthlyRate: 4.543507294763337
+              gpuMonthlyRate: 0
+              monthlyCPUCoreHours: 766.5
+              monthlyRAMByteHours: 1.14819072e+12
+              monthlyGPUHours: 0
+            costChange:
+              totalMonthlyRate: 92.839483652409793
+              cpuMonthlyRate: 24.295976357646456
+              ramMonthlyRate: 4.543507294763337
+              gpuMonthlyRate: 0
+              monthlyCPUCoreHours: 766.5
+              monthlyRAMByteHours: 1.14819072e+12
+              monthlyGPUHours: 0
+    - name: budget
+      variable:
+        value:
+          spendLimit: 100.0
+          currentSpend: 73.0
+    # Calculate the budget that remains from the window by subtracting the currentSpend from the spendLimit.
+    - name: remainingbudget
+      variable:
+        jmesPath: subtract(`{{budget.spendLimit}}`,`{{budget.currentSpend}}`)
+    validate:
+      # Need to improve this by rounding.
+      message: "This Deployment, which costs ${{ predictedcost }} to run for a month, will overrun the remaining budget of ${{ remainingbudget }}. Please seek approval."
+      deny:
+        conditions:
+          all:
+          - key: "{{ predictedcost }}"
+            operator: GreaterThan
+            value: "{{ remainingbudget }}"

test/conformance/chainsaw/deferred/dependencies/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: enforce-company-budget
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/deferred/foreach/01-apply.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: apply
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: manifests.yaml
+  - assert:
+      file: policy-assert.yaml

test/conformance/chainsaw/deferred/foreach/02-testcase.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: testcase
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: cm.yaml
+  - assert:
+      file: cm-assert.yaml

test/conformance/chainsaw/deferred/foreach/README.md

@@ -0,0 +1,11 @@
+## Description
+
+This test checks for deferred variable substitutions in foreach loops
+
+## Expected Behavior
+
+The CM should be created with three new entries
+
+## Reference Issues
+
+https://github.com/kyverno/kyverno/issues/7532

test/conformance/chainsaw/deferred/foreach/cm-assert.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: testcase-7fki3-resource
+data:
+  from_loop_1: AAA
+  from_loop_2: AAA
+  from_loop_3: AAA

test/conformance/chainsaw/deferred/foreach/cm.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: testcase-7fki3-resource

test/conformance/chainsaw/deferred/foreach/manifests.yaml

@@ -0,0 +1,44 @@
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: testcase-7fki3
+spec:
+  admission: true
+  background: false
+  rules:
+  - context:
+    - name: var1
+      variable:
+        value: AAA
+    match:
+      all:
+      - resources:
+          kinds:
+          - v1/ConfigMap
+          names:
+          - testcase-7fki3-resource
+    mutate:
+      foreach:
+      - list: '[''dummy'']'
+        patchStrategicMerge:
+          data:
+            from_loop_1: '{{ var1 || ''!!!variable not resolved!!!'' }}'
+      - list: '[''dummy'']'
+        patchStrategicMerge:
+          data:
+            from_loop_2: '{{ var1 || ''!!!variable not resolved!!!'' }}'
+      - list: '[''dummy'']'
+        patchStrategicMerge:
+          data:
+            from_loop_3: '{{ var1 || ''!!!variable not resolved!!!'' }}'
+    name: mutate1
+    preconditions:
+      all:
+      - key: '{{ request.operation }}'
+        operator: AllIn
+        value:
+        - CREATE
+        - UPDATE
+  schemaValidation: false
+  validationFailureAction: Enforce

test/conformance/chainsaw/deferred/foreach/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: testcase-7fki3
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/deferred/recursive/01-policy.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: policy
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: policy.yaml
+  - assert:
+      file: policy-assert.yaml

test/conformance/chainsaw/deferred/recursive/02-resource.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: resource
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: resource.yaml
+  - assert:
+      file: resource-assert.yaml

test/conformance/chainsaw/deferred/recursive/README.md

@@ -0,0 +1,7 @@
+## Description
+
+This test checks for handling of variable dependencies with the same name with deferred lookups in a foreach
+
+## Expected Behavior
+
+The configmap should create fine and contain `one: one` in the data.

test/conformance/chainsaw/deferred/recursive/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: one
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/deferred/recursive/policy.yaml

@@ -0,0 +1,26 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: one
+spec:
+  validationFailureAction: Enforce
+  rules:
+  - name: one
+    match:
+      all:
+      - resources:
+          kinds:
+          - v1/ConfigMap
+    context:
+    - name: one
+      variable:
+        value: one
+    - name: one
+      variable:
+        jmesPath: one
+    mutate:
+      foreach:
+      - list: "['dummy']"
+        patchStrategicMerge:
+          data:
+            one: "{{ one }}"

test/conformance/chainsaw/deferred/recursive/resource-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: one
+data:
+  one: one

test/conformance/chainsaw/deferred/recursive/resource.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: one

test/conformance/chainsaw/deferred/resolve-overriden-variable/01-policy.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: policy
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: policy.yaml
+  - assert:
+      file: policy-assert.yaml

test/conformance/chainsaw/deferred/resolve-overriden-variable/02-resource.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: resource
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: resource.yaml
+  - assert:
+      file: resource-assert.yaml

test/conformance/chainsaw/deferred/resolve-overriden-variable/README.md

@@ -0,0 +1,9 @@
+## Description
+
+This test checks for handling of variable dependencies with the same name:
+- the same name is used twice in the rule context
+- the same name is also used in a foreach context
+
+## Expected Behavior
+
+The configmap should create fine and contain `one: one` in the data.

test/conformance/chainsaw/deferred/resolve-overriden-variable/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: one
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/deferred/resolve-overriden-variable/policy.yaml

@@ -0,0 +1,33 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: one
+spec:
+  validationFailureAction: Enforce
+  rules:
+  - name: one
+    match:
+      all:
+      - resources:
+          kinds:
+          - v1/ConfigMap
+    context:
+    - name: foo
+      variable:
+        value: foo
+    - name: one
+      variable:
+        jmesPath: foo
+    - name: foo
+      variable:
+        value: baz
+    mutate:
+      foreach:
+      - list: "['dummy']"
+        context:
+        - name: foo
+          variable:
+            value: bar
+        patchStrategicMerge:
+          data:
+            one: "{{ one }}"

test/conformance/chainsaw/deferred/resolve-overriden-variable/resource-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: one
+data:
+  one: foo

test/conformance/chainsaw/deferred/resolve-overriden-variable/resource.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: one

test/conformance/chainsaw/deferred/two-rules/01-policy.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: policy
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: policy.yaml
+  - assert:
+      file: policy-assert.yaml

test/conformance/chainsaw/deferred/two-rules/02-resource.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: resource
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      check: null
+      file: resource.yaml
+  - assert:
+      file: resource-assert.yaml

test/conformance/chainsaw/deferred/two-rules/README.md

@@ -0,0 +1,13 @@
+## Description
+
+This test checks that variables don't leak from one rule to the next.
+The second rule tries to use a variable from the first rule, it should not find it.
+
+## Expected Behavior
+
+The configmap creates fine with the data:
+```yaml
+data:
+    one: test
+    two: "null"
+```

test/conformance/chainsaw/deferred/two-rules/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: one
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/deferred/two-rules/policy.yaml

@@ -0,0 +1,35 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: one
+spec:
+  validationFailureAction: Enforce
+  rules:
+  - name: one
+    match:
+      all:
+      - resources:
+          kinds:
+          - v1/ConfigMap
+    context:
+    - name: var
+      variable:
+        value: test
+    mutate:
+      foreach:
+      - list: "['dummy']"
+        patchStrategicMerge:
+          data:
+            one: "{{ to_string(var) }}"
+  - name: two
+    match:
+      all:
+      - resources:
+          kinds:
+          - v1/ConfigMap
+    mutate:
+      foreach:
+      - list: "['dummy']"
+        patchStrategicMerge:
+          data:
+            two: "{{ to_string(var) }}"

test/conformance/chainsaw/deferred/two-rules/resource-assert.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: one
+data:
+  one: test
+  two: "null"

test/conformance/chainsaw/deferred/two-rules/resource.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: one

test/conformance/chainsaw/exceptions/allows-rejects-creation/01-policy.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: policy
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      file: policy.yaml
+  - assert:
+      file: policy-assert.yaml

test/conformance/chainsaw/exceptions/allows-rejects-creation/02-exception.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: exception
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      file: exception.yaml

test/conformance/chainsaw/exceptions/allows-rejects-creation/03-configmap.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: configmap
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      file: configmap-allowed.yaml
+  - apply:
+      check:
+        (error != null): true
+      file: configmap-rejected.yaml
+  - assert:
+      file: configmap-allowed.yaml
+  - error:
+      file: configmap-rejected.yaml

test/conformance/chainsaw/exceptions/allows-rejects-creation/README.md

@@ -0,0 +1,13 @@
+## Description
+
+This test creates a policy, a policy exception and tries to create a couple configmaps.
+The policy exception is configured to apply only to the `emergency` configmap.
+The `emergency` configmap is expected to create fine while other configmaps creations should fail.
+
+## Steps
+
+1.  - Create a cluster policy
+    - Assert the policy becomes ready
+1.  - Create a policy exception for the cluster policy created above, configured to apply to configmap named `emergency`
+1.  - Try to create a confimap named `emergency`, expecting the creation to succeed
+    - Try to create a confimap named `foo`, expecting the creation to fail

test/conformance/chainsaw/exceptions/allows-rejects-creation/configmap-allowed.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: emergency

test/conformance/chainsaw/exceptions/allows-rejects-creation/configmap-rejected.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: foo

test/conformance/chainsaw/exceptions/allows-rejects-creation/exception.yaml

@@ -0,0 +1,16 @@
+apiVersion: kyverno.io/v2beta1
+kind: PolicyException
+metadata:
+  name: mynewpolex
+spec:
+  exceptions:
+    - policyName: require-labels
+      ruleNames:
+        - require-team
+  match:
+    any:
+      - resources:
+          kinds:
+            - ConfigMap
+          names:
+            - emergency

test/conformance/chainsaw/exceptions/allows-rejects-creation/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/exceptions/allows-rejects-creation/policy.yaml

@@ -0,0 +1,20 @@
+apiVersion: kyverno.io/v2beta1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+spec:
+  validationFailureAction: Enforce
+  background: false
+  rules:
+    - name: require-team
+      match:
+        any:
+          - resources:
+              kinds:
+                - ConfigMap
+      validate:
+        message: 'The label `team` is required.'
+        pattern:
+          metadata:
+            labels:
+              team: '?*'

test/conformance/chainsaw/exceptions/applies-to-delete/01-policy.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: policy
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      file: policy.yaml
+  - assert:
+      file: policy-assert.yaml

test/conformance/chainsaw/exceptions/applies-to-delete/02-exception.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: exception
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      file: exception.yaml

test/conformance/chainsaw/exceptions/applies-to-delete/03-namespace.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: namespace
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      file: namespace.yaml

test/conformance/chainsaw/exceptions/applies-to-delete/04-deployment.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: deployment
+spec:
+  timeouts: {}
+  try:
+  - apply:
+      file: deployment.yaml

test/conformance/chainsaw/exceptions/applies-to-delete/05-delete.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: delete
+spec:
+  timeouts: {}
+  try:
+  - delete:
+      apiVersion: apps/v1
+      kind: Deployment
+      name: test-dpl1
+      namespace: reza-dev

test/conformance/chainsaw/exceptions/applies-to-delete/README.md

@@ -0,0 +1,13 @@
+## Description
+
+This test creates a policy, a policy exception and tries to create a deployment violating the policy.
+The deployment is then deleted.
+
+
+## Expected Behavior
+
+Both creation and deletion should be accepted, the exception applies to all operations.
+
+## Reference Issue(s)
+
+7423

test/conformance/chainsaw/exceptions/applies-to-delete/deployment.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-dpl1
+  namespace: reza-dev
+spec:
+  selector:
+    matchLabels:
+      app: test-dpl1
+  template:
+    metadata:
+      labels:
+        app: test-dpl1
+    spec:
+      securityContext:
+        seccompProfile:
+          type: Unconfined
+      containers:
+      - name: test-dpl1
+        image: busybox:1.35.0
+        command:
+        - sleep
+        - "infinity"

test/conformance/chainsaw/exceptions/applies-to-delete/exception.yaml

@@ -0,0 +1,19 @@
+apiVersion: kyverno.io/v2beta1
+kind: PolicyException
+metadata:
+  name: delta-exception
+  namespace: kyverno
+spec:
+  exceptions:
+  - policyName: psa
+    ruleNames:
+    - "*"
+  match:
+    any:
+    - resources:
+        kinds:
+        - Deployment
+        namespaces:
+        - reza-dev
+        names:
+        - test-dpl1*

test/conformance/chainsaw/exceptions/applies-to-delete/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: reza-dev

test/conformance/chainsaw/exceptions/applies-to-delete/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: psa
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready