feat: use the check block for checking CLI output in chainsaw tests (#9616)

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>

test/conformance/chainsaw/flags/standard/emit-events/chainsaw-test.yaml

@@ -33,7 +33,9 @@ spec:
   - name: step-05
     try:
     - script:
-        content: "if kubectl logs deployment/kyverno-admission-controller -n kyverno
-          | grep \"reason=\\\"PolicyViolation\\\"\" \nthen \n    echo \"Test succeeded.
-          PolicyViolation event was not created.\"\n    exit 0\nelse \n    echo \"Tested
-          failed. PolicyViolation event should have been created.\"\n    exit 1\nfi\n"
+        content: kubectl logs deployment/kyverno-admission-controller -n kyverno
+        check:
+          # This check ensures that the string 'reason="PolicyViolation"' is found
+          # in stdout or else fails
+          (contains($stdout, 'reason=\"PolicyViolation\"')): true
+

test/conformance/chainsaw/generate/clusterpolicy/cornercases/cpol-data-trigger-not-present/chainsaw-test.yaml

@@ -46,5 +46,8 @@ spec:
   - name: step-07
     try:
     - script:
-        content: "if kubectl get updaterequests -n kyverno 2>&1 | grep -q 'No resources found in kyverno namespace.'\nthen \n  exit 0 \nelse \n  exit
-          1\nfi\n"
+        content: kubectl get updaterequests -n kyverno
+        check:
+          # This check ensures the contents of stderr are exactly as shown.
+          ($stderr): |-
+            No resources found in kyverno namespace.

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/delete-trigger-namespace/chainsaw-test.yaml

@@ -37,5 +37,8 @@ spec:
   - name: step-07
     try:
     - script:
-        content: "if kubectl get updaterequests -n kyverno 2>&1 | grep -q 'No resources found in kyverno namespace.'\nthen \n  exit 0 \nelse \n  exit
-          1\nfi\n"
+        content: kubectl get updaterequests -n kyverno
+        check:
+          # This check ensures the contents of stderr are exactly as shown.
+          ($stderr): |-
+            No resources found in kyverno namespace.

test/conformance/chainsaw/mutate/clusterpolicy/standard/existing/multiple-trigger-resources/chainsaw-test.yaml

@@ -34,5 +34,8 @@ spec:
   - name: step-07
     try:
     - script:
-        content: "if kubectl get updaterequests -n kyverno 2>&1 | grep -q 'No resources found in kyverno namespace.'\nthen \n  exit 0 \nelse \n  exit
-          1\nfi\n"
+        content: kubectl get updaterequests -n kyverno
+        check:
+          # This check ensures the contents of stderr are exactly as shown.
+          ($stderr): |-
+            No resources found in kyverno namespace.

test/conformance/chainsaw/validate/clusterpolicy/cornercases/cel-messages-upon-resource-failure/chainsaw-test.yaml

@@ -14,7 +14,7 @@ spec:
   - name: step-02
     try:
     - script:
-        content: "if kubectl apply -f pod-fail.yaml 2>&1 | grep -q 'host-port-pods:
-          hostPort must either be unset or set to 0' \nthen \n    echo \"Test succeeded.
-          The message is displayed.\"\n    exit 0\nelse \n    echo \"Test failed.
-          The message isn't found.\"\n    exit 1\nfi\n"
+        content: kubectl apply -f pod-fail.yaml
+        check:
+          # This check below ensures that the string 'hostPort must either be unset or set to 0' is found in stderr or else fails
+          (contains($stderr, 'hostPort must either be unset or set to 0')): true

test/conformance/chainsaw/validate/clusterpolicy/standard/audit/background-match-clusterRoles/chainsaw-test.yaml

@@ -8,10 +8,10 @@ spec:
   - name: step-01
     try:
     - script:
-        content: "if kubectl apply -f manifests.yaml 2>&1 | grep -q 'invalid variable
-          used' \nthen \n    echo \"Test succeeded. The phrase 'invalid variable used'
-          is found.\"\n    exit 0\nelse \n    echo \"Test failed. The phrase 'invalid
-          variable used' has not been found.\"\n    exit 1\nfi\n"
+        content: kubectl apply -f manifests.yaml
+        check:
+          # This check below ensures that the string 'invalid variable used' is found in stderr or else fails
+          (contains($stderr, 'invalid variable used')): true
   - name: step-02
     try:
     - error:

test/conformance/chainsaw/validate/clusterpolicy/standard/audit/background-match-roles/chainsaw-test.yaml

@@ -8,10 +8,10 @@ spec:
   - name: step-01
     try:
     - script:
-        content: "if kubectl apply -f manifests.yaml 2>&1 | grep -q 'invalid variable
-          used' \nthen \n    echo \"Test succeeded. The phrase 'invalid variable used'
-          is found.\"\n    exit 0\nelse \n    echo \"Test failed. The phrase 'invalid
-          variable used' has not been found.\"\n    exit 1\nfi\n"
+        content: kubectl apply -f manifests.yaml
+        check:
+          # This check below ensures that the string 'invalid variable used' is found in stderr or else fails
+          (contains($stderr, 'invalid variable used')): true
   - name: step-02
     try:
     - error:

test/conformance/chainsaw/validate/clusterpolicy/standard/audit/background-vars-roles/chainsaw-test.yaml

@@ -8,11 +8,10 @@ spec:
   - name: step-01
     try:
     - script:
-        content: "if kubectl apply -f manifests.yaml 2>&1 | grep -q 'variable {{request.roles}}
-          is not allowed' \nthen \n    echo \"Test succeeded. The phrase 'variable
-          {{request.roles}} is not allowed' is found.\"\n    exit 0\nelse \n    echo
-          \"Test failed. The phrase 'variable {{request.roles}} is not allowed' has
-          not been found.\"\n    exit 1\nfi\n"
+        content: kubectl apply -f manifests.yaml
+        check:
+          # This check below ensures that the string 'variable {{request.roles}} is not allowed' is found in stderr or else fails
+          (contains($stderr, 'variable {{request.roles}} is not allowed')): true
   - name: step-02
     try:
     - error:

test/conformance/chainsaw/validate/clusterpolicy/standard/audit/background-vars-serviceAccountName/chainsaw-test.yaml

@@ -8,11 +8,10 @@ spec:
   - name: step-01
     try:
     - script:
-        content: "if kubectl apply -f manifests.yaml 2>&1 | grep -q 'variable {{serviceAccountName}}
-          is not allowed' \nthen \n    echo \"Test succeeded. The phrase 'variable
-          {{serviceAccountName}} is not allowed' is found.\"\n    exit 0\nelse \n
-          \   echo \"Test failed. The phrase 'variable {{serviceAccountName}} is not
-          allowed' has not been found.\"\n    exit 1\nfi\n"
+        content: kubectl apply -f manifests.yaml
+        check:
+          # This check below ensures that the string 'variable {{serviceAccountName}} is not allowed' is found in stderr or else fails
+          (contains($stderr, 'variable {{serviceAccountName}} is not allowed')): true
   - name: step-02
     try:
     - error:

test/conformance/chainsaw/validate/clusterpolicy/standard/audit/background-vars-userInfo/chainsaw-test.yaml

@@ -8,11 +8,10 @@ spec:
   - name: step-01
     try:
     - script:
-        content: "if kubectl apply -f manifests.yaml 2>&1 | grep -q 'variable {{request.userInfo}}
-          is not allowed' \nthen \n    echo \"Test succeeded. The phrase 'variable
-          {{request.userInfo}} is not allowed' is found.\"\n    exit 0\nelse \n    echo
-          \"Test failed. The phrase 'variable {{request.userInfo}} is not allowed'
-          has not been found.\"\n    exit 1\nfi\n"
+        content: kubectl apply -f manifests.yaml
+        check:
+          # This check below ensures that the string 'variable {{request.userInfo}} is not allowed' is found in stderr or else fails
+          (contains($stderr, 'variable {{request.userInfo}} is not allowed')): true
   - name: step-02
     try:
     - error:

test/conformance/chainsaw/validate/clusterpolicy/standard/enforce/block-pod-exec-requests/chainsaw-test.yaml

@@ -18,8 +18,7 @@ spec:
   - name: step-02
     try:
     - script:
-        content: "if kubectl -n test-validate exec nginx -it -- sh 2>&1 | grep -q
-          \"Exec'ing into Pods protected with the label 'exec=false' is forbidden\"
-          \nthen \n  echo \"Tested failed. Exec Request was not blocked.\"\n  exit
-          1 \nelse \n  echo \"Test succeeded. Exec Request was blocked.\"\n  exit
-          0\nfi\n"
+        content: kubectl -n test-validate exec nginx -it -- sh
+        check:
+          # This check below ensures that the string "Exec'ing into Pods protected with the label 'exec=false' is forbidden" isn't found in stderr or else fails
+          (contains($stderr, 'Exec\'ing into Pods protected with the label \'exec=false\' is forbidden')): false

test/conformance/chainsaw/validate/clusterpolicy/standard/enforce/bypass-with-policy-exception/chainsaw-test.yaml

@@ -24,8 +24,7 @@ spec:
   - name: step-02
     try:
     - script:
-        content: "if kubectl scale deployment nginx-test --replicas=1 -n test-validate
-          2>&1 | grep -q 'validation error: nginx-test needs to have 2 replicas' \nthen
-          \n  echo \"Test failed. Resource was blocked from scaling.\"\n  exit 1\nelse
-          \n  echo \"Tested succeeded. Resource was allowed to scale.\"\n  exit 0
-          \nfi\n"
+        content: kubectl scale deployment nginx-test --replicas=1 -n test-validate
+        check:
+          # This check below ensures that the string "nginx-test needs to have 2 replicas" isn't found in stderr or else fails
+          (contains($stderr, 'nginx-test needs to have 2 replicas')): false

test/conformance/chainsaw/validate/clusterpolicy/standard/enforce/operator-allnotin-01/chainsaw-test.yaml

@@ -14,15 +14,10 @@ spec:
   - name: step-02
     try:
     - script:
-        content: |
-          if kubectl apply -f resource.yaml 2>&1 | grep -q "label ''app.kubernetes.io/name'' is required"
-          then
-            echo "Test succeeded. Resource was blocked from label."
-            exit 0
-          else
-            echo "Tested failed. Resource was allowed to create."
-            exit 1
-          fi
+        content: kubectl apply -f resource.yaml
+        check:
+          # This check below ensures that the string "label ''app.kubernetes.io/name'' is required" is found in stderr or else fails
+          (contains($stderr, 'label \'\'app.kubernetes.io/name\'\' is required')): true
   - name: step-03
     try:
     - apply:

test/conformance/chainsaw/validate/clusterpolicy/standard/enforce/scaling-with-kubectl-scale/chainsaw-test.yaml

@@ -20,8 +20,7 @@ spec:
   - name: step-02
     try:
     - script:
-        content: "if kubectl scale deployment nginx-test --replicas=1 -n test-validate
-          2>&1 | grep -q 'validation error: nginx-test needs to have 2 replicas' \nthen
-          \n  echo \"Test succeeded. Resource was blocked from scaling.\"\n  exit
-          0\nelse \n  echo \"Tested failed. Resource was allowed to scale.\"\n  exit
-          1 \nfi\n"
+        content: kubectl scale deployment nginx-test --replicas=1 -n test-validate
+        check:
+          # This check below ensures that the string "validation error: nginx-test needs to have 2 replicas" isn't found in stderr or else fails
+          (contains($stderr, 'nginx-test needs to have 2 replicas')): true