chore: enable all chainsaw tests (#9012)

* chore: enable all chainsaw tests

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* bump chainsaw

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* bump chainsaw

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fixes

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Co-authored-by: Mariam Fahmy <mariam.fahmy@nirmata.com>

.github/workflows/conformance.yaml

@@ -184,17 +184,17 @@ jobs:
           - events
           - exceptions
           - filter
-          # - generate/clusterpolicy
+          - generate/clusterpolicy
-          # - generate/policy
+          - generate/policy
-          # - generate/validation
+          - generate/validation
-          # - mutate
+          - mutate
-          # - policy-validation
+          - policy-validation
           - rangeoperators
           - rbac
-          # - reports
+          - reports
-          # - validate
+          - validate
-          # - verify-manifests
+          - verify-manifests
-          # - verifyImages
+          - verifyImages
           - webhooks
     needs: prepare-images
     name: chainsaw - ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
@@ -236,7 +236,7 @@ jobs:
       - name: Install Chainsaw
         uses: kyverno/chainsaw/.github/actions/install@c08ae80bc45546987edff33212bf33161c84ce59 # v0.0.6
         with:
-          release: v0.0.6
+          release: v0.0.7-alpha.2
       - name: Test with Chainsaw
         shell: bash
         env:

test/conformance/chainsaw/_config/common.yaml

@@ -7,8 +7,8 @@ spec:
     assert: 90s
     cleanup: 150s
     error: 90s
-    exec: 90s
+    exec: 150s
   parallel: 1
   fullName: true
-  failFast: true
+  failFast: false
   excludeTestRegex: '_.+'

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create-patchesJson6902/04-cleanup.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: cleanup
+spec:
+  timeouts: {}
+  try:
+  - command:
+      args:
+      - delete
+      - -f
+      - 01-manifests.yaml
+      - --force
+      - --wait=true
+      - --ignore-not-found=true
+      entrypoint: kubectl

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create-patchesJson6902/cleanup.yaml

@@ -1,4 +0,0 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-commands:
-  - command: kubectl delete -f 01-manifests.yaml --force --wait=true --ignore-not-found=true

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create/04-cleanup.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: cleanup
+spec:
+  timeouts: {}
+  try:
+  - command:
+      args:
+      - delete
+      - -f
+      - 01-manifests.yaml
+      - --force
+      - --wait=true
+      - --ignore-not-found=true
+      entrypoint: kubectl

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create/cleanup.yaml

@@ -1,4 +0,0 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-commands:
-  - command: kubectl delete -f 01-manifests.yaml --force --wait=true --ignore-not-found=true

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-delete/04-cleanup.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: cleanup
+spec:
+  timeouts: {}
+  try:
+  - command:
+      args:
+      - delete
+      - -f
+      - 01-manifests.yaml
+      - --force
+      - --wait=true
+      - --ignore-not-found=true
+      entrypoint: kubectl

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-delete/cleanup.yaml

@@ -1,4 +0,0 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-commands:
-  - command: kubectl delete -f 01-manifests.yaml --force --wait=true --ignore-not-found=true

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-update/04-cleanup.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: cleanup
+spec:
+  timeouts: {}
+  try:
+  - command:
+      args:
+      - delete
+      - -f
+      - 01-manifests.yaml
+      - --force
+      - --wait=true
+      - --ignore-not-found=true
+      entrypoint: kubectl

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-update/cleanup.yaml

@@ -1,4 +0,0 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-commands:
-  - command: kubectl delete -f 01-manifests.yaml --force --wait=true --ignore-not-found=true

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/onpolicyupdate/basic-create-policy/04-cleanup.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  creationTimestamp: null
+  name: cleanup
+spec:
+  timeouts: {}
+  try:
+  - command:
+      args:
+      - delete
+      - -f
+      - 01-manifests.yaml
+      - --force
+      - --wait=true
+      - --ignore-not-found=true
+      entrypoint: kubectl

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/onpolicyupdate/basic-create-policy/cleanup.yaml

@@ -1,4 +0,0 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-commands:
-  - command: kubectl delete -f 01-manifests.yaml --force --wait=true --ignore-not-found=true

test/conformance/chainsaw/mutate/refactor/foreach/remove-multiple-elements-in-descending-order/pod-error.yaml

@@ -6,4 +6,4 @@ spec:
   containers:
   - name: busybox
     image: busybox:1.35
-    env: null
+    (env != null): true

test/conformance/chainsaw/mutate/refactor/nested-foreach/remove-all-env-vars/pod-error.yaml

@@ -6,7 +6,7 @@ spec:
   containers:
   - name: busybox-1
     image: busybox:1.35
-    env: null
+    (env != null): true
   - name: busybox-2
     image: busybox:1.35
-    env: null
+    (env != null): true

test/conformance/chainsaw/policy-validation/cluster-policy/policy-exceptions-disabled/01-policy_exception.yaml

@@ -3,7 +3,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: TestStep
 metadata:
   creationTimestamp: null
-  name: policy_exception
+  name: policy-exception
 spec:
   timeouts: {}
   try:

test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/clusterscoped/02-namespaceConstraint.yaml

@@ -3,7 +3,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: TestStep
 metadata:
   creationTimestamp: null
-  name: namespaceConstraint
+  name: namespace-constraint
 spec:
   timeouts: {}
   try:

test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/match-clusterscoped-resource/02-nameConstraint.yaml

@@ -3,7 +3,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: TestStep
 metadata:
   creationTimestamp: null
-  name: nameConstraint
+  name: name-constraint
 spec:
   timeouts: {}
   try:

test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/set-paramref-namespace/03-replicaLimit.yaml

@@ -3,7 +3,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: TestStep
 metadata:
   creationTimestamp: null
-  name: replicaLimit
+  name: replica-limit
 spec:
   timeouts: {}
   try:

test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/unset-paramref-namespace/03-replicaLimit.yaml

@@ -3,7 +3,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: TestStep
 metadata:
   creationTimestamp: null
-  name: replicaLimit
+  name: replica-limit
 spec:
   timeouts: {}
   try:

test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/01-policy.yaml

@@ -0,0 +1,10 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: policy
+spec:
+  try:
+  - apply:
+      file: policy.yaml
+  - assert:
+      file: policy-assert.yaml

test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/02-assert.yaml

@@ -1,11 +0,0 @@
----
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: preconditions
-status:
-  conditions:
-  - reason: Succeeded
-    status: "True"
-    type: Ready
-

test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/02-test.yaml

@@ -1,11 +1,9 @@
----
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: TestStep
 metadata:
   creationTimestamp: null
   name: test
 spec:
-  timeouts: {}
   try:
   - apply:
       file: pod-good.yaml

test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/03-cleanup.yaml

@@ -1,13 +1,13 @@
----
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: TestStep
 metadata:
-  creationTimestamp: null
   name: cleanup
 spec:
-  timeouts: {}
   try:
   - delete:
       apiVersion: v1
       kind: Pod
       name: test
+    timeout: 1m
+  - apply:
+      file: policy-2.yaml

test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/policy-assert.yaml

@@ -1,4 +1,3 @@
----
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:
@@ -8,4 +7,3 @@ status:
   - reason: Succeeded
     status: "True"
     type: Ready
-

test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/policy.yaml

@@ -1,4 +1,3 @@
----
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:

test/conformance/chainsaw/verifyImages/clusterpolicy/cornercases/multiple-attestors/chainsaw-test.yaml

@@ -0,0 +1,7 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: multiple-attestors
+spec:
+  timeouts:
+    apply: 90s

test/conformance/chainsaw/verifyImages/clusterpolicy/standard/rollback-image-verification/03-test.yaml

@@ -1,5 +1,11 @@
-apiVersion: kuttl.dev/v1beta1
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: TestStep
-commands:
+metadata:
-  - command: kubectl -n verify-images rollout undo deployment nginx-deployment
+  creationTimestamp: null
-    namespaced: true
+  name: test
+spec:
+  timeouts: {}
+  try:
+  - script:
+      content: kubectl -n verify-images rollout undo deployment nginx-deployment

test/conformance/chainsaw/verifyImages/clusterpolicy/standard/update-multi-containers/02-resource.yaml

@@ -9,5 +9,7 @@ spec:
   try:
   - apply:
       file: resource-v1.yaml
+    timeout: 90s
   - apply:
       file: resource-v2.yaml
+    timeout: 90s