[Bug] [CLI] Restore warn-exit-code functionality for apply command (#9828)

* Restore warn-exite-code functionality for apply command

Signed-off-by: Matt Veitas <mveitas@gmail.com>

* Nove error handling

Signed-off-by: Matt Veitas <mveitas@gmail.com>

* Uncomment println statement

Signed-off-by: Matt Veitas <mveitas@gmail.com>

* Fixing linting

Signed-off-by: Matt Veitas <mveitas@gmail.com>

* Adding conformance tets for cli apply command with warn-exit-code

Signed-off-by: Matt Veitas <mveitas@gmail.com>

* Update path to kubectl-kyverno binary

Signed-off-by: Matt Veitas <mveitas@gmail.com>

* Add prepare-cli as needed dependency

Signed-off-by: Matt Veitas <mveitas@gmail.com>

* feat: install kubectl-kyverno in standard conformance tests

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: update chainsaw config

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: move CLI chainsaw tests to a separate action

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: CLI path

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: name

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: add chainsaw flag '--no-cluster'

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: CLI name

Signed-off-by: ShutingZhao <shuting@nirmata.com>

---------

Signed-off-by: Matt Veitas <mveitas@gmail.com>
Signed-off-by: ShutingZhao <shuting@nirmata.com>
Signed-off-by: shuting <shuting@nirmata.com>
Co-authored-by: ShutingZhao <shuting@nirmata.com>

.github/workflows/conformance.yaml

@@ -114,7 +114,8 @@ jobs:
           - ^verify-manifests$
           - ^verifyImages$
           - ^webhooks$
-    needs: prepare-images
+    needs:
+    - prepare-images
     name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
     steps:
       - name: Checkout
@@ -946,20 +947,38 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       packages: read
-    needs: prepare-cli
+    strategy:
+      fail-fast: false
+      matrix:
+        tests:
+          - ^cli$
+    needs:
+    - prepare-cli
+    name: ${{ matrix.tests }} - chainsaw
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      # install tools
       - name: Download kyverno CLI archive
         uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
         with:
           name: kubectl-kyverno
+      - name: Install chainsaw
+        uses: kyverno/action-install-chainsaw@3bf0752f44d348d859fefa022f113bda6a24a1ae # v0.1.7
       - name: Install Kyverno CLI
         shell: bash
         run: |
           set -e
           chmod +x kubectl-kyverno && mv kubectl-kyverno ./cmd/cli/kubectl-kyverno/kyverno
           echo "$PWD/cmd/cli/kubectl-kyverno" >> $GITHUB_PATH
+      # run tests
+      - name: Test with Chainsaw
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -e
+          cd ./test/conformance/chainsaw && chainsaw test --include-test-regex '^chainsaw$/${{ matrix.tests }}' --config ../../../.chainsaw.yaml --no-cluster
       - name: Fix test files
         shell: bash
         run: |

cmd/cli/kubectl-kyverno/_testdata/apply/test-2/policy.yaml

@@ -0,0 +1,28 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: test-policy
+  annotations:
+    policies.kyverno.io/title: "Lol Security Standards"
+    policies.kyverno.io/category: "Lol Security Standards"
+    policies.kyverno.io/severity: "high"
+    policies.kyverno.io/subject: "Pod"
+spec:
+  background: true
+  failurePolicy: Fail
+  rules:
+    - name: restrict-lol-annotation
+      match:
+        any:
+        - resources:
+            kinds:
+            - Pod
+      validate:
+        message: >-
+          If "lol" annotation is present then
+          its value can be only one of "such lol", "much annotation".
+        pattern:
+          # syntax refdoc: https://kyverno.io/docs/writing-policies/validate/#anchors
+          =(metadata):
+            =(annotations):
+              =(lol): such lol | much annotation

cmd/cli/kubectl-kyverno/_testdata/apply/test-2/resources.yaml

@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: lol
+  name: i-will-fail-the-policy-check
+spec:
+  selector:
+    matchLabels:
+      app: lol
+  template:
+    metadata:
+      labels:
+        app: lol
+      annotations:
+        lol: not much
+    spec:
+      containers:
+      - image: woot
+        name: woot

cmd/cli/kubectl-kyverno/commands/apply/command.go

@@ -95,7 +95,7 @@ func Command() *cobra.Command {
 			} else {
 				printViolations(out, rc)
 			}
-			return exit(rc, applyCommandConfig.warnExitCode, applyCommandConfig.warnNoPassed)
+			return exit(out, rc, applyCommandConfig.warnExitCode, applyCommandConfig.warnNoPassed)
 		},
 	}
 	cmd.Flags().StringSliceVarP(&applyCommandConfig.ResourcePaths, "resource", "r", []string{}, "Path to resource files")
@@ -465,15 +465,29 @@ func printViolations(out io.Writer, rc *processor.ResultCounts) {
 	fmt.Fprintf(out, "\npass: %d, fail: %d, warn: %d, error: %d, skip: %d \n", rc.Pass(), rc.Fail(), rc.Warn(), rc.Error(), rc.Skip())
 }
 
-func exit(rc *processor.ResultCounts, warnExitCode int, warnNoPassed bool) error {
+type WarnExitCodeError struct {
+	ExitCode int
+}
+
+func (w WarnExitCodeError) Error() string {
+	return fmt.Sprintf("exit as warnExitCode is %d", w.ExitCode)
+}
+
+func exit(out io.Writer, rc *processor.ResultCounts, warnExitCode int, warnNoPassed bool) error {
 	if rc.Fail() > 0 {
 		return fmt.Errorf("exit as there are policy violations")
 	} else if rc.Error() > 0 {
 		return fmt.Errorf("exit as there are policy errors")
 	} else if rc.Warn() > 0 && warnExitCode != 0 {
-		return fmt.Errorf("exit as warnExitCode is %d", warnExitCode)
+		fmt.Printf("exit as warnExitCode is %d", warnExitCode)
+		return WarnExitCodeError{
+			ExitCode: warnExitCode,
+		}
 	} else if rc.Pass() == 0 && warnNoPassed {
-		return fmt.Errorf("exit as no objects satisfied policy")
+		fmt.Println(out, "exit as no objects satisfied policy")
+		return WarnExitCodeError{
+			ExitCode: warnExitCode,
+		}
 	}
 	return nil
 }

cmd/cli/kubectl-kyverno/commands/apply/command_test.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -114,6 +115,7 @@ func Test_Apply(t *testing.T) {
 				ResourcePaths: []string{"../../../../../test/resources/pod_with_latest_tag.yaml"},
 				PolicyReport:  true,
 				AuditWarn:     true,
+				warnExitCode:  3,
 			},
 			stdinFile: "../../../../../test/best_practices/disallow_latest_tag.yaml",
 			expectedPolicyReports: []policyreportv1alpha2.PolicyReport{{
@@ -480,6 +482,29 @@ func TestCommandWithInvalidFlag(t *testing.T) {
 	assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(out)))
 }
 
+func TestCommandWarnExitCode(t *testing.T) {
+	var warnExitCode = 3
+
+	cmd := Command()
+	cmd.SetArgs([]string{
+		"../../_testdata/apply/test-2/policy.yaml",
+		"--resource",
+		"../../_testdata/apply/test-2/resources.yaml",
+		"--audit-warn",
+		"--warn-exit-code",
+		strconv.Itoa(warnExitCode),
+	})
+	err := cmd.Execute()
+	if err != nil {
+		switch e := err.(type) {
+		case WarnExitCodeError:
+			assert.Equal(t, warnExitCode, e.ExitCode)
+		default:
+			assert.Fail(t, "Expecting WarnExitCodeError")
+		}
+	}
+}
+
 func TestCommandHelp(t *testing.T) {
 	cmd := Command()
 	assert.NotNil(t, cmd)

cmd/cli/kubectl-kyverno/main.go

@@ -6,6 +6,7 @@ import (
 	"os"
 
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/commands"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/commands/apply"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/experimental"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/log"
 	"github.com/spf13/cobra"
@@ -18,7 +19,12 @@ func main() {
 		os.Exit(1)
 	}
 	if err := cmd.Execute(); err != nil {
-		os.Exit(1)
+		switch e := err.(type) {
+		case apply.WarnExitCodeError:
+			os.Exit(e.ExitCode)
+		default:
+			os.Exit(1)
+		}
 	}
 }
 

test/conformance/chainsaw/cli/apply/warn-exit-code/chainsaw-test.yaml

@@ -0,0 +1,12 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: test-name
+spec:
+  steps:
+    - try:
+      - script:
+          content: |
+            kyverno apply policy.yaml --resource resource.yaml --audit-warn --warn-exit-code 3
+          check:
+            ($error): "exit status 3"

test/conformance/chainsaw/cli/apply/warn-exit-code/policy.yaml

@@ -0,0 +1,28 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: test-policy
+  annotations:
+    policies.kyverno.io/title: "Lol Security Standards"
+    policies.kyverno.io/category: "Lol Security Standards"
+    policies.kyverno.io/severity: "high"
+    policies.kyverno.io/subject: "Pod"
+spec:
+  background: true
+  failurePolicy: Fail
+  rules:
+    - name: restrict-lol-annotation
+      match:
+        any:
+          - resources:
+              kinds:
+                - Pod
+      validate:
+        message: >-
+          If "lol" annotation is present then
+          its value can be only one of "such lol", "much annotation".
+        pattern:
+          # syntax refdoc: https://kyverno.io/docs/writing-policies/validate/#anchors
+          =(metadata):
+            =(annotations):
+              =(lol): such lol | much annotation

test/conformance/chainsaw/cli/apply/warn-exit-code/resource.yaml

@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: lol
+  name: i-will-fail-the-policy-check
+spec:
+  selector:
+    matchLabels:
+      app: lol
+  template:
+    metadata:
+      labels:
+        app: lol
+      annotations:
+        lol: not much
+    spec:
+      containers:
+        - image: woot
+          name: woot