diff --git a/.github/workflows/conformance.yaml b/.github/workflows/conformance.yaml index 9f6b7b5f56..bafbb574d2 100644 --- a/.github/workflows/conformance.yaml +++ b/.github/workflows/conformance.yaml @@ -184,17 +184,17 @@ jobs: - events - exceptions - filter - # - generate/clusterpolicy - # - generate/policy - # - generate/validation - # - mutate - # - policy-validation + - generate/clusterpolicy + - generate/policy + - generate/validation + - mutate + - policy-validation - rangeoperators - rbac - # - reports - # - validate - # - verify-manifests - # - verifyImages + - reports + - validate + - verify-manifests + - verifyImages - webhooks needs: prepare-images name: chainsaw - ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }} @@ -236,7 +236,7 @@ jobs: - name: Install Chainsaw uses: kyverno/chainsaw/.github/actions/install@c08ae80bc45546987edff33212bf33161c84ce59 # v0.0.6 with: - release: v0.0.6 + release: v0.0.7-alpha.2 - name: Test with Chainsaw shell: bash env: diff --git a/test/conformance/chainsaw/_config/common.yaml b/test/conformance/chainsaw/_config/common.yaml index aa67bacd60..48c0aa3124 100755 --- a/test/conformance/chainsaw/_config/common.yaml +++ b/test/conformance/chainsaw/_config/common.yaml @@ -7,8 +7,8 @@ spec: assert: 90s cleanup: 150s error: 90s - exec: 90s + exec: 150s parallel: 1 fullName: true - failFast: true + failFast: false excludeTestRegex: '_.+' diff --git a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create-patchesJson6902/04-cleanup.yaml b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create-patchesJson6902/04-cleanup.yaml new file mode 100644 index 0000000000..2eccd177b1 --- /dev/null +++ b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create-patchesJson6902/04-cleanup.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: TestStep +metadata: + creationTimestamp: null + name: cleanup +spec: + timeouts: {} + try: + - command: + args: + - delete + - -f + - 01-manifests.yaml + - --force + - --wait=true + - --ignore-not-found=true + entrypoint: kubectl diff --git a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create-patchesJson6902/cleanup.yaml b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create-patchesJson6902/cleanup.yaml deleted file mode 100644 index 15c3c49051..0000000000 --- a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create-patchesJson6902/cleanup.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - command: kubectl delete -f 01-manifests.yaml --force --wait=true --ignore-not-found=true \ No newline at end of file diff --git a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create/04-cleanup.yaml b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create/04-cleanup.yaml new file mode 100644 index 0000000000..2eccd177b1 --- /dev/null +++ b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create/04-cleanup.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: TestStep +metadata: + creationTimestamp: null + name: cleanup +spec: + timeouts: {} + try: + - command: + args: + - delete + - -f + - 01-manifests.yaml + - --force + - --wait=true + - --ignore-not-found=true + entrypoint: kubectl diff --git a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create/cleanup.yaml b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create/cleanup.yaml deleted file mode 100644 index 15c3c49051..0000000000 --- a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-create/cleanup.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - command: kubectl delete -f 01-manifests.yaml --force --wait=true --ignore-not-found=true \ No newline at end of file diff --git a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-delete/04-cleanup.yaml b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-delete/04-cleanup.yaml new file mode 100644 index 0000000000..2eccd177b1 --- /dev/null +++ b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-delete/04-cleanup.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: TestStep +metadata: + creationTimestamp: null + name: cleanup +spec: + timeouts: {} + try: + - command: + args: + - delete + - -f + - 01-manifests.yaml + - --force + - --wait=true + - --ignore-not-found=true + entrypoint: kubectl diff --git a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-delete/cleanup.yaml b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-delete/cleanup.yaml deleted file mode 100644 index 15c3c49051..0000000000 --- a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-delete/cleanup.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - command: kubectl delete -f 01-manifests.yaml --force --wait=true --ignore-not-found=true \ No newline at end of file diff --git a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-update/04-cleanup.yaml b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-update/04-cleanup.yaml new file mode 100644 index 0000000000..2eccd177b1 --- /dev/null +++ b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-update/04-cleanup.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: TestStep +metadata: + creationTimestamp: null + name: cleanup +spec: + timeouts: {} + try: + - command: + args: + - delete + - -f + - 01-manifests.yaml + - --force + - --wait=true + - --ignore-not-found=true + entrypoint: kubectl diff --git a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-update/cleanup.yaml b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-update/cleanup.yaml deleted file mode 100644 index 15c3c49051..0000000000 --- a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/basic-update/cleanup.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - command: kubectl delete -f 01-manifests.yaml --force --wait=true --ignore-not-found=true \ No newline at end of file diff --git a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/onpolicyupdate/basic-create-policy/04-cleanup.yaml b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/onpolicyupdate/basic-create-policy/04-cleanup.yaml new file mode 100644 index 0000000000..2eccd177b1 --- /dev/null +++ b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/onpolicyupdate/basic-create-policy/04-cleanup.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: TestStep +metadata: + creationTimestamp: null + name: cleanup +spec: + timeouts: {} + try: + - command: + args: + - delete + - -f + - 01-manifests.yaml + - --force + - --wait=true + - --ignore-not-found=true + entrypoint: kubectl diff --git a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/onpolicyupdate/basic-create-policy/cleanup.yaml b/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/onpolicyupdate/basic-create-policy/cleanup.yaml deleted file mode 100644 index 15c3c49051..0000000000 --- a/test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/onpolicyupdate/basic-create-policy/cleanup.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - command: kubectl delete -f 01-manifests.yaml --force --wait=true --ignore-not-found=true \ No newline at end of file diff --git a/test/conformance/chainsaw/mutate/refactor/foreach/remove-multiple-elements-in-descending-order/pod-error.yaml b/test/conformance/chainsaw/mutate/refactor/foreach/remove-multiple-elements-in-descending-order/pod-error.yaml index 07c7eaf5cc..285e223653 100644 --- a/test/conformance/chainsaw/mutate/refactor/foreach/remove-multiple-elements-in-descending-order/pod-error.yaml +++ b/test/conformance/chainsaw/mutate/refactor/foreach/remove-multiple-elements-in-descending-order/pod-error.yaml @@ -6,4 +6,4 @@ spec: containers: - name: busybox image: busybox:1.35 - env: null \ No newline at end of file + (env != null): true diff --git a/test/conformance/chainsaw/mutate/refactor/nested-foreach/remove-all-env-vars/pod-error.yaml b/test/conformance/chainsaw/mutate/refactor/nested-foreach/remove-all-env-vars/pod-error.yaml index 24bc3167b7..8eaaa37a91 100644 --- a/test/conformance/chainsaw/mutate/refactor/nested-foreach/remove-all-env-vars/pod-error.yaml +++ b/test/conformance/chainsaw/mutate/refactor/nested-foreach/remove-all-env-vars/pod-error.yaml @@ -6,7 +6,7 @@ spec: containers: - name: busybox-1 image: busybox:1.35 - env: null + (env != null): true - name: busybox-2 image: busybox:1.35 - env: null + (env != null): true diff --git a/test/conformance/chainsaw/policy-validation/cluster-policy/policy-exceptions-disabled/01-policy_exception.yaml b/test/conformance/chainsaw/policy-validation/cluster-policy/policy-exceptions-disabled/01-policy_exception.yaml index b8b70f8198..ce36eaa553 100644 --- a/test/conformance/chainsaw/policy-validation/cluster-policy/policy-exceptions-disabled/01-policy_exception.yaml +++ b/test/conformance/chainsaw/policy-validation/cluster-policy/policy-exceptions-disabled/01-policy_exception.yaml @@ -3,7 +3,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: TestStep metadata: creationTimestamp: null - name: policy_exception + name: policy-exception spec: timeouts: {} try: diff --git a/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/clusterscoped/02-namespaceConstraint.yaml b/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/clusterscoped/02-namespaceConstraint.yaml index 24667595a3..fc937fcebf 100644 --- a/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/clusterscoped/02-namespaceConstraint.yaml +++ b/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/clusterscoped/02-namespaceConstraint.yaml @@ -3,7 +3,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: TestStep metadata: creationTimestamp: null - name: namespaceConstraint + name: namespace-constraint spec: timeouts: {} try: diff --git a/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/match-clusterscoped-resource/02-nameConstraint.yaml b/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/match-clusterscoped-resource/02-nameConstraint.yaml index 5d7235b5f1..df09445ef6 100644 --- a/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/match-clusterscoped-resource/02-nameConstraint.yaml +++ b/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/match-clusterscoped-resource/02-nameConstraint.yaml @@ -3,7 +3,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: TestStep metadata: creationTimestamp: null - name: nameConstraint + name: name-constraint spec: timeouts: {} try: diff --git a/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/set-paramref-namespace/03-replicaLimit.yaml b/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/set-paramref-namespace/03-replicaLimit.yaml index b4ef0abb56..b37fe68e2c 100644 --- a/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/set-paramref-namespace/03-replicaLimit.yaml +++ b/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/set-paramref-namespace/03-replicaLimit.yaml @@ -3,7 +3,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: TestStep metadata: creationTimestamp: null - name: replicaLimit + name: replica-limit spec: timeouts: {} try: diff --git a/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/unset-paramref-namespace/03-replicaLimit.yaml b/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/unset-paramref-namespace/03-replicaLimit.yaml index b4ef0abb56..b37fe68e2c 100644 --- a/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/unset-paramref-namespace/03-replicaLimit.yaml +++ b/test/conformance/chainsaw/validate/clusterpolicy/standard/cel/parameter-resources/namespaced/unset-paramref-namespace/03-replicaLimit.yaml @@ -3,7 +3,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: TestStep metadata: creationTimestamp: null - name: replicaLimit + name: replica-limit spec: timeouts: {} try: diff --git a/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/01-policy.yaml b/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/01-policy.yaml new file mode 100644 index 0000000000..744135ecd0 --- /dev/null +++ b/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/01-policy.yaml @@ -0,0 +1,10 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: TestStep +metadata: + name: policy +spec: + try: + - apply: + file: policy.yaml + - assert: + file: policy-assert.yaml diff --git a/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/02-assert.yaml b/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/02-assert.yaml deleted file mode 100644 index 48b630df69..0000000000 --- a/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/02-assert.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: kyverno.io/v1 -kind: ClusterPolicy -metadata: - name: preconditions -status: - conditions: - - reason: Succeeded - status: "True" - type: Ready - diff --git a/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/02-test.yaml b/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/02-test.yaml index 0621641386..392f3e48a5 100644 --- a/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/02-test.yaml +++ b/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/02-test.yaml @@ -1,11 +1,9 @@ ---- apiVersion: chainsaw.kyverno.io/v1alpha1 kind: TestStep metadata: creationTimestamp: null name: test spec: - timeouts: {} try: - apply: file: pod-good.yaml diff --git a/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/03-cleanup.yaml b/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/03-cleanup.yaml index c7717f8e22..8c0d1dc659 100644 --- a/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/03-cleanup.yaml +++ b/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/03-cleanup.yaml @@ -1,13 +1,13 @@ ---- apiVersion: chainsaw.kyverno.io/v1alpha1 kind: TestStep metadata: - creationTimestamp: null name: cleanup spec: - timeouts: {} try: - delete: apiVersion: v1 kind: Pod name: test + timeout: 1m + - apply: + file: policy-2.yaml diff --git a/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/03-manifests.yaml b/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/policy-2.yaml similarity index 100% rename from test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/03-manifests.yaml rename to test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/policy-2.yaml diff --git a/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/01-assert.yaml b/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/policy-assert.yaml similarity index 96% rename from test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/01-assert.yaml rename to test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/policy-assert.yaml index 48b630df69..199f8746dc 100644 --- a/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/01-assert.yaml +++ b/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/policy-assert.yaml @@ -1,4 +1,3 @@ ---- apiVersion: kyverno.io/v1 kind: ClusterPolicy metadata: @@ -8,4 +7,3 @@ status: - reason: Succeeded status: "True" type: Ready - diff --git a/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/01-manifests.yaml b/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/policy.yaml similarity index 99% rename from test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/01-manifests.yaml rename to test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/policy.yaml index 507a7e11e2..66a01a5123 100644 --- a/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/01-manifests.yaml +++ b/test/conformance/chainsaw/validate/clusterpolicy/standard/variables/lazyload/conditions/policy.yaml @@ -1,4 +1,3 @@ ---- apiVersion: kyverno.io/v1 kind: ClusterPolicy metadata: diff --git a/test/conformance/chainsaw/verifyImages/clusterpolicy/cornercases/multiple-attestors/chainsaw-test.yaml b/test/conformance/chainsaw/verifyImages/clusterpolicy/cornercases/multiple-attestors/chainsaw-test.yaml new file mode 100644 index 0000000000..178176c3b4 --- /dev/null +++ b/test/conformance/chainsaw/verifyImages/clusterpolicy/cornercases/multiple-attestors/chainsaw-test.yaml @@ -0,0 +1,7 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: multiple-attestors +spec: + timeouts: + apply: 90s diff --git a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/rollback-image-verification/03-test.yaml b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/rollback-image-verification/03-test.yaml index 831f02fed7..1c1bd30fee 100644 --- a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/rollback-image-verification/03-test.yaml +++ b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/rollback-image-verification/03-test.yaml @@ -1,5 +1,11 @@ -apiVersion: kuttl.dev/v1beta1 +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 kind: TestStep -commands: - - command: kubectl -n verify-images rollout undo deployment nginx-deployment - namespaced: true \ No newline at end of file +metadata: + creationTimestamp: null + name: test +spec: + timeouts: {} + try: + - script: + content: kubectl -n verify-images rollout undo deployment nginx-deployment diff --git a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/update-multi-containers/02-resource.yaml b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/update-multi-containers/02-resource.yaml index 43390639ff..a66ed9f429 100644 --- a/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/update-multi-containers/02-resource.yaml +++ b/test/conformance/chainsaw/verifyImages/clusterpolicy/standard/update-multi-containers/02-resource.yaml @@ -9,5 +9,7 @@ spec: try: - apply: file: resource-v1.yaml + timeout: 90s - apply: file: resource-v2.yaml + timeout: 90s