chore: fix chainsaw tests cleanup timeout (#9028)

* chore: fix chainsaw tests cleanup timeout

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* bump chainsaw

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* grace

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* checks

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* bump chainsaw

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix deletes

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix exceptions

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

.github/workflows/conformance.yaml

@@ -147,7 +147,7 @@ jobs:
       - name: Install Chainsaw
         uses: kyverno/chainsaw/.github/actions/install@c08ae80bc45546987edff33212bf33161c84ce59 # v0.0.6
         with:
-          release: v0.0.7-alpha.3
+          release: v0.0.7-alpha.5
       - name: Test with Chainsaw
         shell: bash
         env:
@@ -217,7 +217,7 @@ jobs:
       - name: Install Chainsaw
         uses: kyverno/chainsaw/.github/actions/install@c08ae80bc45546987edff33212bf33161c84ce59 # v0.0.6
         with:
-          release: v0.0.7-alpha.3
+          release: v0.0.7-alpha.5
       - name: Test with Chainsaw
         shell: bash
         env:
@@ -287,7 +287,7 @@ jobs:
       - name: Install Chainsaw
         uses: kyverno/chainsaw/.github/actions/install@c08ae80bc45546987edff33212bf33161c84ce59 # v0.0.6
         with:
-          release: v0.0.7-alpha.3
+          release: v0.0.7-alpha.5
       - name: Test with Chainsaw
         shell: bash
         env:
@@ -359,7 +359,7 @@ jobs:
       - name: Install Chainsaw
         uses: kyverno/chainsaw/.github/actions/install@c08ae80bc45546987edff33212bf33161c84ce59 # v0.0.6
         with:
-          release: v0.0.7-alpha.3
+          release: v0.0.7-alpha.5
       - name: Test with Chainsaw
         shell: bash
         env:
@@ -429,7 +429,7 @@ jobs:
       - name: Install Chainsaw
         uses: kyverno/chainsaw/.github/actions/install@c08ae80bc45546987edff33212bf33161c84ce59 # v0.0.6
         with:
-          release: v0.0.7-alpha.3
+          release: v0.0.7-alpha.5
       - name: Test with Chainsaw
         shell: bash
         env:
@@ -505,7 +505,7 @@ jobs:
       - name: Install Chainsaw
         uses: kyverno/chainsaw/.github/actions/install@c08ae80bc45546987edff33212bf33161c84ce59 # v0.0.6
         with:
-          release: v0.0.7-alpha.3
+          release: v0.0.7-alpha.5
       - name: Test with Chainsaw
         shell: bash
         env:
@@ -580,7 +580,7 @@ jobs:
       - name: Install Chainsaw
         uses: kyverno/chainsaw/.github/actions/install@c08ae80bc45546987edff33212bf33161c84ce59 # v0.0.6
         with:
-          release: v0.0.7-alpha.3
+          release: v0.0.7-alpha.5
       - name: Test with Chainsaw
         shell: bash
         env:
@@ -674,7 +674,7 @@ jobs:
       - name: Install Chainsaw
         uses: kyverno/chainsaw/.github/actions/install@c08ae80bc45546987edff33212bf33161c84ce59 # v0.0.6
         with:
-          release: v0.0.7-alpha.3
+          release: v0.0.7-alpha.5
       - name: Test with Chainsaw
         shell: bash
         env:
@@ -748,7 +748,7 @@ jobs:
       - name: Install Chainsaw
         uses: kyverno/chainsaw/.github/actions/install@c08ae80bc45546987edff33212bf33161c84ce59 # v0.0.6
         with:
-          release: v0.0.7-alpha.3
+          release: v0.0.7-alpha.5
       - name: Test with Chainsaw
         shell: bash
         env:

test/conformance/chainsaw/_config/common.yaml

@@ -5,9 +5,9 @@ metadata:
 spec:
   timeouts:
     assert: 90s
-    cleanup: 150s
     error: 90s
   parallel: 1
   fullName: true
   failFast: false
   excludeTestRegex: '_.+'
+  forceTerminationGracePeriod: 5s

test/conformance/chainsaw/cleanup/validation/cron-format/03-invalidpolicy.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: invalidpolicy.yaml

test/conformance/chainsaw/cleanup/validation/no-user-info-in-match/01-cleanuppolicy.yaml

@@ -8,14 +8,17 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: cleanuppolicy-with-subjects.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: cleanuppolicy-with-roles.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: cleanuppolicy-with-clusterroles.yaml

test/conformance/chainsaw/cleanup/validation/not-supported-attributes-in-context/01-cleanup-policy.yaml

@@ -8,10 +8,12 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: cleanuppolicy-with-image-registry.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: cleanuppolicy-with-configmap.yaml

test/conformance/chainsaw/deferred/dependencies/02-testcase.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: deploy.yaml

test/conformance/chainsaw/events/policy/policy-violation/02-resource.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: resource.yaml

test/conformance/chainsaw/exceptions/allows-rejects-creation/03-configmap.yaml

@@ -10,8 +10,9 @@ spec:
   - apply:
       file: configmap-allowed.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: configmap-rejected.yaml
   - assert:
       file: configmap-allowed.yaml

test/conformance/chainsaw/exceptions/applies-to-delete/05-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: apps/v1
+      ref:
-      kind: Deployment
+        apiVersion: apps/v1
-      name: test-dpl1
+        kind: Deployment
-      namespace: reza-dev
+        name: test-dpl1
+        namespace: reza-dev

test/conformance/chainsaw/exceptions/background-mode/standard/01-exception.yaml

@@ -10,8 +10,9 @@ spec:
   - apply:
       file: exception-allowed.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: exception-rejected.yaml
   - assert:
       file: exception-allowed.yaml

test/conformance/chainsaw/exceptions/conditions/03-deployment.yaml

@@ -10,6 +10,11 @@ spec:
   - apply:
       file: good-deployment.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: bad-deployment.yaml
+  finally:
+  - script:
+      content: sleep 5
+    timeout: 10s

test/conformance/chainsaw/exceptions/conditions/04-sleep.yaml

@@ -1,13 +0,0 @@
----
-apiVersion: chainsaw.kyverno.io/v1alpha1
-kind: TestStep
-metadata:
-  creationTimestamp: null
-  name: sleep
-spec:
-  timeouts: {}
-  try:
-  - command:
-      args:
-      - "4"
-      entrypoint: sleep

test/conformance/chainsaw/exceptions/only-for-specific-user/03-configmap.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: configmap.yaml

test/conformance/chainsaw/exceptions/with-wildcard/03-configmap.yaml

@@ -10,8 +10,9 @@ spec:
   - apply:
       file: configmap-allowed.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: configmap-rejected.yaml
   - assert:
       file: configmap-allowed.yaml

test/conformance/chainsaw/filter/exclude/sa/no-wildcard/02-resource.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: resource.yaml

test/conformance/chainsaw/filter/exclude/sa/wildcard/02-resource.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: resource.yaml

test/conformance/chainsaw/filter/exclude/user/no-wildcard/block/02-resource.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: resource.yaml

test/conformance/chainsaw/filter/exclude/user/wildcard/block/02-resource.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: resource.yaml

test/conformance/chainsaw/filter/match/user/no-wildcard/block/02-resource.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: resource.yaml

test/conformance/chainsaw/filter/match/user/wildcard/block/02-resource.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: resource.yaml

test/conformance/chainsaw/flags/standard/emit-events/03-resource.yaml

@@ -10,6 +10,7 @@ spec:
   - apply:
       file: resource.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: resource-fail.yaml

test/conformance/chainsaw/generate/clusterpolicy/cornercases/clone-list-sync-same-trigger-source-delete-source/03-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: Secret
+        apiVersion: v1
-      name: mysecret
+        kind: Secret
-      namespace: clone-list-sync-same-trigger-source-trigger-ns
+        name: mysecret
+        namespace: clone-list-sync-same-trigger-source-trigger-ns

test/conformance/chainsaw/generate/clusterpolicy/cornercases/clone-source-name-exceeds-63-characters/04-delete-source.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: Secret
+        apiVersion: v1
-      name: regcredregcredregcredregcredregcredregcredregcredregcredregcredregcredregcredregcredregcredregcred
+        kind: Secret
-      namespace: default
+        name: regcredregcredregcredregcredregcredregcredregcredregcredregcredregcredregcredregcredregcredregcred
+        namespace: default

test/conformance/chainsaw/generate/clusterpolicy/cornercases/clone-sync-same-trigger-source-delete-source/03-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: Secret
+        apiVersion: v1
-      name: mysecret
+        kind: Secret
-      namespace: clone-sync-same-trigger-source-trigger-ns
+        name: mysecret
+        namespace: clone-sync-same-trigger-source-trigger-ns

test/conformance/chainsaw/generate/clusterpolicy/cornercases/cpol-clone-create-on-trigger-deletion/02-delete.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: Namespace
+        apiVersion: v1
-      name: cpol-clone-create-on-trigger-deletion-trigger-ns
+        kind: Namespace
+        name: cpol-clone-create-on-trigger-deletion-trigger-ns

test/conformance/chainsaw/generate/clusterpolicy/cornercases/cpol-clone-sync-reinstall-policy/04-delete.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: kyverno.io/v1
+      ref:
-      kind: ClusterPolicy
+        apiVersion: kyverno.io/v1
-      name: cpol-clone-sync-reinstall-policy
+        kind: ClusterPolicy
+        name: cpol-clone-sync-reinstall-policy

test/conformance/chainsaw/generate/clusterpolicy/cornercases/cpol-data-create-on-trigger-deletion/02-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: test-org
+        kind: ConfigMap
-      namespace: cpol-create-on-trigger-deletion-ns
+        name: test-org
+        namespace: cpol-create-on-trigger-deletion-ns

test/conformance/chainsaw/generate/clusterpolicy/cornercases/trigger-resource-name-exceeds-63-characters/04-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: my-configmapmy-configmapmy-configmapmy-configmapmy-configmapmy-configmap
+        kind: ConfigMap
-      namespace: trigger-resource-name-exceeds-63-characters-ns
+        name: my-configmapmy-configmapmy-configmapmy-configmapmy-configmapmy-configmap
+        namespace: trigger-resource-name-exceeds-63-characters-ns

test/conformance/chainsaw/generate/clusterpolicy/standard/clone/nosync/cpol-clone-nosync-delete-downstream/04-delete-secret.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: Secret
+        apiVersion: v1
-      name: regcred
+        kind: Secret
-      namespace: cpol-clone-nosync-delete-downstream-ns
+        name: regcred
+        namespace: cpol-clone-nosync-delete-downstream-ns

test/conformance/chainsaw/generate/clusterpolicy/standard/clone/nosync/cpol-clone-nosync-delete-policy/03-removepolicy.yaml

@@ -10,6 +10,7 @@ spec:
   - assert:
       file: check.yaml
   - delete:
-      apiVersion: kyverno.io/v2beta1
+      ref:
-      kind: ClusterPolicy
+        apiVersion: kyverno.io/v2beta1
-      name: cpol-nosync-clone-delete-policy
+        kind: ClusterPolicy
+        name: cpol-nosync-clone-delete-policy

test/conformance/chainsaw/generate/clusterpolicy/standard/clone/nosync/cpol-clone-nosync-delete-source/03-deletesource.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: Secret
+        apiVersion: v1
-      name: regcred
+        kind: Secret
+        name: regcred

test/conformance/chainsaw/generate/clusterpolicy/standard/clone/nosync/cpol-clone-nosync-delete-trigger/03-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: test-org
+        kind: ConfigMap
-      namespace: cpol-clone-nosync-delete-trigger-ns
+        name: test-org
+        namespace: cpol-clone-nosync-delete-trigger-ns

test/conformance/chainsaw/generate/clusterpolicy/standard/clone/sync/cpol-clone-list-sync-delete-source/03-delete.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: Namespace
+        apiVersion: v1
-      name: cpol-clone-list-sync-delete-source-trigger-ns-1
+        kind: Namespace
+        name: cpol-clone-list-sync-delete-source-trigger-ns-1

test/conformance/chainsaw/generate/clusterpolicy/standard/clone/sync/cpol-clone-sync-delete-downstream/04-delete-secret.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: Secret
+        apiVersion: v1
-      name: regcred
+        kind: Secret
-      namespace: cpol-clone-sync-delete-downstream-ns
+        name: regcred
+        namespace: cpol-clone-sync-delete-downstream-ns

test/conformance/chainsaw/generate/clusterpolicy/standard/clone/sync/cpol-clone-sync-delete-policy/04-delete.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: kyverno.io/v2beta1
+      ref:
-      kind: ClusterPolicy
+        apiVersion: kyverno.io/v2beta1
-      name: cpol-clone-sync-delete-policy
+        kind: ClusterPolicy
+        name: cpol-clone-sync-delete-policy

test/conformance/chainsaw/generate/clusterpolicy/standard/clone/sync/cpol-clone-sync-delete-source/03-deletesource.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: Secret
+        apiVersion: v1
-      name: regcred
+        kind: Secret
-      namespace: cpol-clone-sync-delete-source-ns
+        name: regcred
+        namespace: cpol-clone-sync-delete-source-ns

test/conformance/chainsaw/generate/clusterpolicy/standard/clone/sync/cpol-clone-sync-delete-trigger/03-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: test-org
+        kind: ConfigMap
-      namespace: cpol-clone-sync-delete-trigger-ns
+        name: test-org
+        namespace: cpol-clone-sync-delete-trigger-ns

test/conformance/chainsaw/generate/clusterpolicy/standard/data/nosync/cpol-data-nosync-delete-downstream/04-downstream-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: zk-kafka-address
+        kind: ConfigMap
-      namespace: cpol-data-nosync-delete-downstream-ns
+        name: zk-kafka-address
+        namespace: cpol-data-nosync-delete-downstream-ns

test/conformance/chainsaw/generate/clusterpolicy/standard/data/nosync/cpol-data-nosync-delete-policy/03-delete-policy.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: kyverno.io/v1
+      ref:
-      kind: ClusterPolicy
+        apiVersion: kyverno.io/v1
-      name: cpol-data-nosync-delete-policy-policy
+        kind: ClusterPolicy
+        name: cpol-data-nosync-delete-policy-policy

test/conformance/chainsaw/generate/clusterpolicy/standard/data/nosync/cpol-data-nosync-delete-trigger/04-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: test-org
+        kind: ConfigMap
-      namespace: cpol-data-nosync-delete-trigger-ns
+        name: test-org
+        namespace: cpol-data-nosync-delete-trigger-ns

test/conformance/chainsaw/generate/clusterpolicy/standard/data/sync/cpol-data-sync-delete-downstream/03-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: zk-kafka-address
+        kind: ConfigMap
-      namespace: cpol-data-sync-delete-downstream-ns
+        name: zk-kafka-address
+        namespace: cpol-data-sync-delete-downstream-ns

test/conformance/chainsaw/generate/clusterpolicy/standard/data/sync/cpol-data-sync-delete-one-trigger/03-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: foosource-1
+        kind: ConfigMap
-      namespace: cpol-data-sync-delete-one-trigger-ns
+        name: foosource-1
+        namespace: cpol-data-sync-delete-one-trigger-ns

test/conformance/chainsaw/generate/clusterpolicy/standard/data/sync/cpol-data-sync-delete-policy/04-policy-delete.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: kyverno.io/v1
+      ref:
-      kind: ClusterPolicy
+        apiVersion: kyverno.io/v1
-      name: cpol-data-sync-delete-policy
+        kind: ClusterPolicy
+        name: cpol-data-sync-delete-policy

test/conformance/chainsaw/generate/clusterpolicy/standard/data/sync/cpol-data-sync-delete-trigger/02-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: test-org
+        kind: ConfigMap
-      namespace: cpol-data-sync-delete-trigger-ns
+        name: test-org
+        namespace: cpol-data-sync-delete-trigger-ns

test/conformance/chainsaw/generate/policy/cornercases/pol-clone-create-on-trigger-deletion/02-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: pol-clone-create-on-trigger-deletion-configmap
+        kind: ConfigMap
-      namespace: pol-clone-create-on-trigger-deletion-ns
+        name: pol-clone-create-on-trigger-deletion-configmap
+        namespace: pol-clone-create-on-trigger-deletion-ns

test/conformance/chainsaw/generate/policy/cornercases/pol-data-create-on-trigger-deletion/02-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: test-org
+        kind: ConfigMap
-      namespace: pol-create-on-trigger-deletion-ns
+        name: test-org
+        namespace: pol-create-on-trigger-deletion-ns

test/conformance/chainsaw/generate/policy/standard/clone/nosync/pol-clone-nosync-delete-downstream/04-delete-downstream.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: Secret
+        apiVersion: v1
-      name: newsecret
+        kind: Secret
-      namespace: default
+        name: newsecret
+        namespace: default

test/conformance/chainsaw/generate/policy/standard/clone/nosync/pol-clone-nosync-delete-policy/03-delete-policy.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: kyverno.io/v2beta1
+      ref:
-      kind: Policy
+        apiVersion: kyverno.io/v2beta1
-      name: pol-clone-nosync-delete-policy
+        kind: Policy
-      namespace: default
+        name: pol-clone-nosync-delete-policy
+        namespace: default

test/conformance/chainsaw/generate/policy/standard/clone/nosync/pol-clone-nosync-delete-source/03-delete-source.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: Secret
+        apiVersion: v1
-      name: regcred
+        kind: Secret
-      namespace: default
+        name: regcred
+        namespace: default

test/conformance/chainsaw/generate/policy/standard/clone/nosync/pol-clone-nosync-delete-trigger/03-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: test-org
+        kind: ConfigMap
-      namespace: pol-clone-nosync-delete-trigger-ns
+        name: test-org
+        namespace: pol-clone-nosync-delete-trigger-ns

test/conformance/chainsaw/generate/policy/standard/clone/sync/pol-clone-sync-delete-policy/04-delete-policy.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: kyverno.io/v2beta1
+      ref:
-      kind: Policy
+        apiVersion: kyverno.io/v2beta1
-      name: pol-clone-sync-delete-policy
+        kind: Policy
-      namespace: default
+        name: pol-clone-sync-delete-policy
+        namespace: default

test/conformance/chainsaw/generate/policy/standard/clone/sync/pol-clone-sync-delete-source/03-deletesource.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: Secret
+        apiVersion: v1
-      name: regcred
+        kind: Secret
-      namespace: pol-clone-sync-delete-source
+        name: regcred
+        namespace: pol-clone-sync-delete-source

test/conformance/chainsaw/generate/policy/standard/clone/sync/pol-clone-sync-delete-trigger/03-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: test-org
+        kind: ConfigMap
-      namespace: pol-clone-sync-delete-trigger-ns
+        name: test-org
+        namespace: pol-clone-sync-delete-trigger-ns

test/conformance/chainsaw/generate/policy/standard/data/nosync/pol-data-nosync-create-policy-invalid/01-create.yaml

@@ -10,6 +10,7 @@ spec:
   - apply:
       file: ns.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy.yaml

test/conformance/chainsaw/generate/policy/standard/data/nosync/pol-data-nosync-delete-downstream/03-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: zk-kafka-address
+        kind: ConfigMap
-      namespace: hammer
+        name: zk-kafka-address
+        namespace: hammer

test/conformance/chainsaw/generate/policy/standard/data/nosync/pol-data-nosync-delete-policy/03-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: kyverno.io/v1
+      ref:
-      kind: Policy
+        apiVersion: kyverno.io/v1
-      name: pol-data-nosync-delete-policy-policy
+        kind: Policy
-      namespace: manta
+        name: pol-data-nosync-delete-policy-policy
+        namespace: manta

test/conformance/chainsaw/generate/policy/standard/data/nosync/pol-data-nosync-delete-trigger/04-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: test-org
+        kind: ConfigMap
-      namespace: pol-data-nosync-delete-trigger-ns
+        name: test-org
+        namespace: pol-data-nosync-delete-trigger-ns

test/conformance/chainsaw/generate/policy/standard/data/sync/pol-data-sync-delete-downstream/03-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: zk-kafka-address
+        kind: ConfigMap
-      namespace: exeter
+        name: zk-kafka-address
+        namespace: exeter

test/conformance/chainsaw/generate/policy/standard/data/sync/pol-data-sync-delete-policy/03-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: kyverno.io/v1
+      ref:
-      kind: Policy
+        apiVersion: kyverno.io/v1
-      name: pol-data-sync-delete-policy-policy
+        kind: Policy
-      namespace: manasis
+        name: pol-data-sync-delete-policy-policy
+        namespace: manasis

test/conformance/chainsaw/generate/policy/standard/data/sync/pol-data-sync-delete-trigger/05-delete.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: test-org
+        kind: ConfigMap
-      namespace: pol-data-sync-delete-trigger-ns
+        name: test-org
+        namespace: pol-data-sync-delete-trigger-ns

test/conformance/chainsaw/generate/validation/clusterpolicy/cloneList/02-check.yaml

@@ -10,14 +10,17 @@ spec:
   - apply:
       file: policy-pass.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-fail-1.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-fail-2.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-fail-3.yaml

test/conformance/chainsaw/generate/validation/clusterpolicy/immutable-clone/02-update.yaml

@@ -8,10 +8,12 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-name.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-namespace.yaml

test/conformance/chainsaw/generate/validation/clusterpolicy/immutable-clonelist/02-update.yaml

@@ -8,14 +8,17 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-ns.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-kinds.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-selector.yaml

test/conformance/chainsaw/generate/validation/clusterpolicy/immutable-downstream/02-update.yaml

@@ -8,18 +8,22 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-name.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-apiversion.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-namespace.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-kind.yaml

test/conformance/chainsaw/generate/validation/clusterpolicy/immutable-rule-spec/02-update.yaml

@@ -8,20 +8,24 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-rule-name.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-rule-match.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-rule-exclude.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-rule-preconditions.yaml
   - apply:
       file: update-rule-generate-synchronize.yaml

test/conformance/chainsaw/generate/validation/clusterpolicy/permissions/no-permission/01-fail-no-permission.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy.yaml

test/conformance/chainsaw/generate/validation/clusterpolicy/permissions/no-permission/04-delete.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: kyverno.io/v1
+      ref:
-      kind: ClusterPolicy
+        apiVersion: kyverno.io/v1
-      name: cpol-validate-create-sa-permission
+        kind: ClusterPolicy
+        name: cpol-validate-create-sa-permission

test/conformance/chainsaw/generate/validation/clusterpolicy/permissions/same-kind/03-check.yaml

@@ -12,10 +12,12 @@ spec:
   - apply:
       file: policy-1-subresource.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-2.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-2-subresource.yaml

test/conformance/chainsaw/generate/validation/clusterpolicy/target-namespace-scope/02-check.yaml

@@ -8,12 +8,14 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-fail-1-no-ns-namespaced-target.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-fail-2-ns-cluster-target.yaml
   - apply:
       file: policy-pass-1-ns-namespaced-target.yaml

test/conformance/chainsaw/generate/validation/policy/cloneList/02-check.yaml

@@ -10,14 +10,17 @@ spec:
   - apply:
       file: policy-pass.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-fail-1.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-fail-2.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-fail-3.yaml

test/conformance/chainsaw/generate/validation/policy/immutable-clone/02-update.yaml

@@ -8,10 +8,12 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-namespace.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-name.yaml

test/conformance/chainsaw/generate/validation/policy/immutable-clonelist/02-update.yaml

@@ -8,14 +8,17 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-ns.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-kinds.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-selector.yaml

test/conformance/chainsaw/generate/validation/policy/immutable-downstream/02-update.yaml

@@ -8,18 +8,22 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-name.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-apiversion.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-namespace.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-kind.yaml

test/conformance/chainsaw/generate/validation/policy/immutable-rule-spec/02-update.yaml

@@ -8,20 +8,24 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-rule-name.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-rule-match.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-rule-exclude.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: update-rule-preconditions.yaml
   - apply:
       file: update-rule-generate-synchronize.yaml

test/conformance/chainsaw/generate/validation/policy/permissions/02-fail-no-permission.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy.yaml

test/conformance/chainsaw/generate/validation/policy/target-namespace-scope/02-check.yaml

@@ -10,18 +10,22 @@ spec:
   - apply:
       file: policy-pass.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-fail-0.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-fail-1.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-fail-2.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-fail-3.yaml

test/conformance/chainsaw/generate/validation/policy/target-namespace-scope/03-delete.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: apiextensions.k8s.io/v1
+      ref:
-      kind: CustomResourceDefinition
+        apiVersion: apiextensions.k8s.io/v1
-      name: roles.iam.aws.crossplane.io
+        kind: CustomResourceDefinition
+        name: roles.iam.aws.crossplane.io

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-delete/02-delete-cm.yaml

@@ -8,7 +8,8 @@ spec:
   timeouts: {}
   try:
   - delete:
-      apiVersion: v1
+      ref:
-      kind: ConfigMap
+        apiVersion: v1
-      name: dictionary-2
+        kind: ConfigMap
-      namespace: staging-2
+        name: dictionary-2
+        namespace: staging-2

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/validation/auth-check/cpol-namespace-variable/01-fail-no-permission.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy.yaml

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/validation/auth-check/cpol-standard-auth-check/01-fail-no-permission.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy.yaml

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/validation/mutate-existing-require-targets/01-no-targets-fail.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-no-targets.yaml

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/validation/target-variable-validation/01-policy.yaml

@@ -8,8 +8,9 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-bad.yaml
   - apply:
       file: policy-good.yaml

test/conformance/chainsaw/mutate/policy/standard/existing/validation/auth-check/02-fail-no-permission.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy.yaml

test/conformance/chainsaw/policy-validation/cluster-policy/admission-disabled/01-policy.yaml

@@ -10,14 +10,17 @@ spec:
   - apply:
       file: policy-validate.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-mutate.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-generate.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-verify-image.yaml

test/conformance/chainsaw/policy-validation/cluster-policy/all-disabled/01-policy.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy.yaml

test/conformance/chainsaw/policy-validation/cluster-policy/background-subresource/01-policy.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-1.yaml

test/conformance/chainsaw/policy-validation/cluster-policy/background-subresource/02-policy.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-2.yaml

test/conformance/chainsaw/policy-validation/cluster-policy/background-subresource/03-policy.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-3.yaml

test/conformance/chainsaw/policy-validation/cluster-policy/background-subresource/04-policy.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-4.yaml

test/conformance/chainsaw/policy-validation/cluster-policy/background-subresource/05-policy.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-5.yaml

test/conformance/chainsaw/policy-validation/cluster-policy/invalid-subject-kind/01-policies.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy.yaml

test/conformance/chainsaw/policy-validation/cluster-policy/invalid-timeout/01-policies.yaml

@@ -8,10 +8,12 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-1.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-2.yaml

test/conformance/chainsaw/policy-validation/cluster-policy/policy-exceptions-disabled/02-resource.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: resource.yaml

test/conformance/chainsaw/policy-validation/cluster-policy/target-context/01-policies.yaml

@@ -8,10 +8,12 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-1.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-2.yaml

test/conformance/chainsaw/policy-validation/policy/admission-disabled/01-policy.yaml

@@ -10,10 +10,12 @@ spec:
   - apply:
       file: policy-validate.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-mutate.yaml
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-verify-image.yaml

test/conformance/chainsaw/policy-validation/policy/all-disabled/01-policy.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy.yaml

test/conformance/chainsaw/policy-validation/policy/background-subresource/01-policy.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-1.yaml

test/conformance/chainsaw/policy-validation/policy/background-subresource/02-policy.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-2.yaml

test/conformance/chainsaw/policy-validation/policy/background-subresource/03-policy.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-3.yaml

test/conformance/chainsaw/policy-validation/policy/background-subresource/04-policy.yaml

@@ -8,6 +8,7 @@ spec:
   timeouts: {}
   try:
   - apply:
-      check:
+      expect:
-        (error != null): true
+      - check:
+          ($error != null): true
       file: policy-4.yaml