fix: CLI test command should validate the policy under test (#8387)

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

Makefile

@@ -521,7 +521,7 @@ codegen-cli-docs: $(CLI_BIN) ## Generate CLI docs
 .PHONY: codegen-cli-tests
 codegen-cli-tests: $(CLI_BIN) ## Fix CLI test files
 	@echo Fix CLI test files... >&2
-	@KYVERNO_EXPERIMENTAL=true $(CLI_BIN) fix test ./test/cli --save
+	@KYVERNO_EXPERIMENTAL=true $(CLI_BIN) fix test ./test/cli --save --compress
 
 .PHONY: codegen-docs-all
 codegen-docs-all: codegen-helm-docs codegen-cli-docs codegen-api-docs  ## Generate all docs

cmd/cli/kubectl-kyverno/_testdata/policies/invalid-schema.yaml

@@ -0,0 +1,43 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: pod-requirements
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+    policies.kyverno.io/severity: medium
+    policies.kyverno.io/category: Pod Security Standards (Restricted)
+spec:
+  background: false
+  validationFailureAction: audit
+  rules:
+  - name: pods-require-account
+    match:
+      resources:
+        kinds:
+        - Pod
+      namespaceSelector:
+        matchLabels:
+          istio/rev: "default"
+    validate:
+      message: User pods must include an account for charging
+      pattern:
+        metadata:
+          labels:
+            account: "*?"
+  - name: pods-require-limits
+    match:
+      resources:
+        kinds:
+        - Pod
+    validate:
+      message: CPU and memory resource requests and limits are required for user pods
+      pattern:
+        spec:
+          containers:
+          - resources:
+              requests:
+                memory: "?*"
+                cpu: "?*"
+              limits:
+                memory: "?*"
+                cpu: "?*"

cmd/cli/kubectl-kyverno/policy/load_test.go

@@ -0,0 +1,38 @@
+package policy
+
+import (
+	"testing"
+
+	"github.com/go-git/go-billy/v5"
+)
+
+func TestLoad(t *testing.T) {
+	tests := []struct {
+		name         string
+		fs           billy.Filesystem
+		resourcePath string
+		paths        []string
+		wantErr      bool
+	}{{
+		name:         "cpol-limit-configmap-for-sa",
+		fs:           nil,
+		resourcePath: "",
+		paths:        []string{"../_testdata/policies/cpol-limit-configmap-for-sa.yaml"},
+		wantErr:      false,
+	}, {
+		name:         "invalid-schema",
+		fs:           nil,
+		resourcePath: "",
+		paths:        []string{"../_testdata/policies/invalid-schema.yaml"},
+		wantErr:      true,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, _, err := Load(tt.fs, tt.resourcePath, tt.paths...)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Load() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+		})
+	}
+}

pkg/utils/yaml/loadpolicy.go

@@ -54,9 +54,9 @@ func addPolicy(policies []kyvernov1.PolicyInterface, validatingAdmissionPolicies
 	kind := us.GetKind()
 
 	if strings.Compare(kind, "ValidatingAdmissionPolicy") == 0 {
-		validatingAdmissionPolicy := &v1alpha1.ValidatingAdmissionPolicy{}
+		validatingAdmissionPolicy := v1alpha1.ValidatingAdmissionPolicy{}
 
-		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(us.Object, validatingAdmissionPolicy); err != nil {
+		if err := runtime.DefaultUnstructuredConverter.FromUnstructuredWithValidation(us.Object, &validatingAdmissionPolicy, true); err != nil {
 			return policies, nil, fmt.Errorf("failed to decode policy: %v", err)
 		}
 
@@ -65,7 +65,7 @@ func addPolicy(policies []kyvernov1.PolicyInterface, validatingAdmissionPolicies
 			return policies, validatingAdmissionPolicies, nil
 		}
 
-		validatingAdmissionPolicies = append(validatingAdmissionPolicies, *validatingAdmissionPolicy)
+		validatingAdmissionPolicies = append(validatingAdmissionPolicies, validatingAdmissionPolicy)
 	} else {
 		var policy kyvernov1.PolicyInterface
 		if us.GetKind() == "ClusterPolicy" {
@@ -74,7 +74,7 @@ func addPolicy(policies []kyvernov1.PolicyInterface, validatingAdmissionPolicies
 			policy = &kyvernov1.Policy{}
 		}
 
-		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(us.Object, policy); err != nil {
+		if err := runtime.DefaultUnstructuredConverter.FromUnstructuredWithValidation(us.Object, policy, true); err != nil {
 			return nil, validatingAdmissionPolicies, fmt.Errorf("failed to decode policy: %v", err)
 		}
 

pkg/utils/yaml/loadpolicy_test.go

@@ -314,8 +314,8 @@ spec:
         apiVersions: ["v1"]
         operations:  ["CREATE", "UPDATE"]
         resources:   ["deployments"]
-    validations:
-      - expression: "object.spec.replicas <= 5"
+  validations:
+    - expression: "object.spec.replicas <= 5"
 `),
 		}, validatingAdmissionPolicies: []policy{
 			{"ValidatingAdmissionPolicy", ""},
@@ -337,8 +337,8 @@ spec:
         apiVersions: ["v1"]
         operations:  ["CREATE", "UPDATE"]
         resources:   ["deployments"]
-    validations:
-      - expression: "object.spec.replicas <= 5"
+  validations:
+    - expression: "object.spec.replicas <= 5"
 ---
 apiVersion: kyverno.io/v1
 kind: Policy
@@ -391,8 +391,8 @@ spec:
         apiVersions: ["v1"]
         operations:  ["CREATE", "UPDATE"]
         resources:   ["deployments"]
-    validations:
-      - expression: "object.spec.replicas <= 5"
+  validations:
+    - expression: "object.spec.replicas <= 5"
 ---
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy

test/cli/scenarios_to_cli/other/scenario_mutate_endpoint/policy.yaml

@@ -10,9 +10,9 @@ spec:
         - resources:
             kinds:
             - Endpoints
-          selector:
-            matchLabels:
-              label: test
+            selector:
+              matchLabels:
+                label: test
       mutate:
         patchesJson6902: |-
           [

test/cli/test/deny-modify-platform-label-2/kyverno-test.yaml

@@ -3,7 +3,6 @@ policies:
 - deny-modify-platform-label.yaml
 resources:
 - resource.yaml
-variables: variables.yaml
 results:
 - kind: Role
   policy: deny-modify-platform-label
@@ -15,11 +14,7 @@ results:
   policy: deny-modify-platform-label
   resources:
   - my-role-without-platform
-  result: skip
-  rule: deny-modify-platform-role
-- kind: Role
-  policy: deny-modify-platform-label
-  resources:
   - my-role-with-platform-false
   result: skip
   rule: deny-modify-platform-role
+variables: variables.yaml