added resource lists for test cli (#4082)

Signed-off-by: Tathagata Paul <tathagatapaul7@gmail.com>

cmd/cli/kubectl-kyverno/test/test_command.go

@@ -256,6 +256,8 @@ type TestResults struct {
 	Status policyreportv1alpha2.PolicyResult `json:"status"`
 	// Resource mentions the name of the resource on which the policy is to be applied.
 	Resource string `json:"resource"`
+	// Resources gives us the list of resources on which the policy is going to be applied.
+	Resources []string `json:"resources"`
 	// Kind mentions the kind of the resource on which the policy is to be applied.
 	Kind string `json:"kind"`
 	// Namespace mentions the namespace of the policy which has namespace scope.
@@ -313,7 +315,7 @@ type testFilter struct {
 	enabled  bool
 }
 
-var ftable = []*Table{}
+var ftable = []Table{}
 
 func testCommandExecute(dirPath []string, fileName string, gitBranch string, testCase string) (rc *resultCounts, err error) {
 	var errors []error
@@ -547,41 +549,77 @@ func buildPolicyResults(engineResponses []*response.EngineResponse, testResults
 				test.Policy = userDefinedPolicyName
 			}
 
-			if test.Policy == policyName && test.Resource == resourceName {
-				var resultsKey string
+			if test.Resources != nil {
+				if test.Policy == policyName {
+					// results[].namespace value implict set same as metadata.namespace until and unless
+					// user provides explict values for results[].namespace in test yaml file.
+					if test.Namespace == "" {
+						test.Namespace = resourceNamespace
+						testResults[i].Namespace = resourceNamespace
+					}
+					for _, resource := range test.Resources {
+						if resource == resourceName {
+							var resultsKey string
+							resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, resource)
+							if !util.ContainsString(rules, test.Rule) {
+								if !util.ContainsString(rules, "autogen-"+test.Rule) {
+									if !util.ContainsString(rules, "autogen-cronjob-"+test.Rule) {
+										result.Result = policyreportv1alpha2.StatusSkip
+									} else {
+										testResults[i].AutoGeneratedRule = "autogen-cronjob"
+										test.Rule = "autogen-cronjob-" + test.Rule
+										resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, resource)
+									}
+								} else {
+									testResults[i].AutoGeneratedRule = "autogen"
+									test.Rule = "autogen-" + test.Rule
+									resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, resource)
+								}
 
-				// results[].namespace value implict set same as metadata.namespace until and unless
-				// user provides explict values for results[].namespace in test yaml file.
-				if test.Namespace == "" {
-					test.Namespace = resourceNamespace
-					testResults[i].Namespace = resourceNamespace
+								if results[resultsKey].Result == "" {
+									result.Result = policyreportv1alpha2.StatusSkip
+									results[resultsKey] = result
+								}
+							}
+
+							patchedResourcePath = append(patchedResourcePath, test.PatchedResource)
+							if _, ok := results[resultsKey]; !ok {
+								results[resultsKey] = result
+							}
+						}
+					}
 				}
-				resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, test.Resource)
-				if !util.ContainsString(rules, test.Rule) {
-					if !util.ContainsString(rules, "autogen-"+test.Rule) {
-						if !util.ContainsString(rules, "autogen-cronjob-"+test.Rule) {
-							result.Result = policyreportv1alpha2.StatusSkip
+			}
+			if test.Resource != "" {
+				if test.Policy == policyName && test.Resource == resourceName {
+					var resultsKey string
+					resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, test.Resource)
+					if !util.ContainsString(rules, test.Rule) {
+						if !util.ContainsString(rules, "autogen-"+test.Rule) {
+							if !util.ContainsString(rules, "autogen-cronjob-"+test.Rule) {
+								result.Result = policyreportv1alpha2.StatusSkip
+							} else {
+								testResults[i].AutoGeneratedRule = "autogen-cronjob"
+								test.Rule = "autogen-cronjob-" + test.Rule
+								resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, test.Resource)
+							}
 						} else {
-							testResults[i].AutoGeneratedRule = "autogen-cronjob"
-							test.Rule = "autogen-cronjob-" + test.Rule
+							testResults[i].AutoGeneratedRule = "autogen"
+							test.Rule = "autogen-" + test.Rule
 							resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, test.Resource)
 						}
-					} else {
-						testResults[i].AutoGeneratedRule = "autogen"
-						test.Rule = "autogen-" + test.Rule
-						resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, test.Resource)
+
+						if results[resultsKey].Result == "" {
+							result.Result = policyreportv1alpha2.StatusSkip
+							results[resultsKey] = result
+						}
 					}
 
-					if results[resultsKey].Result == "" {
-						result.Result = policyreportv1alpha2.StatusSkip
+					patchedResourcePath = append(patchedResourcePath, test.PatchedResource)
+					if _, ok := results[resultsKey]; !ok {
 						results[resultsKey] = result
 					}
 				}
-
-				patchedResourcePath = append(patchedResourcePath, test.PatchedResource)
-				if _, ok := results[resultsKey]; !ok {
-					results[resultsKey] = result
-				}
 			}
 
 			for _, rule := range resp.PolicyResponse.Rules {
@@ -796,7 +834,6 @@ func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, isGit bool,
 		}
 		values.Results = filteredResults
 	}
-
 	if len(values.Results) == 0 {
 		return nil
 	}
@@ -913,6 +950,11 @@ func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, isGit bool,
 	filteredResources := []*unstructured.Unstructured{}
 	for _, r := range resources {
 		for _, res := range values.Results {
+			for _, testr := range res.Resources {
+				if r.GetName() == testr {
+					filteredResources = append(filteredResources, r)
+				}
+			}
 			if r.GetName() == res.Resource {
 				filteredResources = append(filteredResources, r)
 				break
@@ -970,7 +1012,6 @@ func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, isGit bool,
 			pvInfos = append(pvInfos, info)
 		}
 	}
-
 	resultsMap, testResults := buildPolicyResults(engineResponses, values.Results, pvInfos, policyResourcePath, fs, isGit)
 	resultErr := printTestResult(resultsMap, testResults, rc)
 	if resultErr != nil {
@@ -982,76 +1023,142 @@ func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, isGit bool,
 
 func printTestResult(resps map[string]policyreportv1alpha2.PolicyReportResult, testResults []TestResults, rc *resultCounts) error {
 	printer := tableprinter.New(os.Stdout)
-	table := []*Table{}
+	table := []Table{}
 	boldGreen := color.New(color.FgGreen).Add(color.Bold)
 	boldRed := color.New(color.FgRed).Add(color.Bold)
 	boldYellow := color.New(color.FgYellow).Add(color.Bold)
 	boldFgCyan := color.New(color.FgCyan).Add(color.Bold)
 
+	countDeprecatedResource := 0
 	for i, v := range testResults {
 		res := new(Table)
 		res.ID = i + 1
 		res.Policy = boldFgCyan.Sprintf(v.Policy)
 		res.Rule = boldFgCyan.Sprintf(v.Rule)
 
-		res.Resource = boldFgCyan.Sprintf(v.Namespace) + "/" + boldFgCyan.Sprintf(v.Kind) + "/" + boldFgCyan.Sprintf(v.Resource)
-		var ruleNameInResultKey string
-		if v.AutoGeneratedRule != "" {
-			ruleNameInResultKey = fmt.Sprintf("%s-%s", v.AutoGeneratedRule, v.Rule)
-		} else {
-			ruleNameInResultKey = v.Rule
-		}
+		if v.Resources != nil {
+			for _, resource := range v.Resources {
+				res.Resource = boldFgCyan.Sprintf(v.Namespace) + "/" + boldFgCyan.Sprintf(v.Kind) + "/" + boldFgCyan.Sprintf(resource)
+				var ruleNameInResultKey string
+				if v.AutoGeneratedRule != "" {
+					ruleNameInResultKey = fmt.Sprintf("%s-%s", v.AutoGeneratedRule, v.Rule)
+				} else {
+					ruleNameInResultKey = v.Rule
+				}
 
-		resultKey := fmt.Sprintf("%s-%s-%s-%s", v.Policy, ruleNameInResultKey, v.Kind, v.Resource)
-		found, _ := isNamespacedPolicy(v.Policy)
-		var ns string
-		ns, v.Policy = getUserDefinedPolicyNameAndNamespace(v.Policy)
-		if found && v.Namespace != "" {
-			resultKey = fmt.Sprintf("%s-%s-%s-%s-%s-%s", ns, v.Policy, ruleNameInResultKey, v.Namespace, v.Kind, v.Resource)
-		} else if found {
-			resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", ns, v.Policy, ruleNameInResultKey, v.Kind, v.Resource)
-			res.Policy = boldFgCyan.Sprintf(ns) + "/" + boldFgCyan.Sprintf(v.Policy)
-			res.Resource = boldFgCyan.Sprintf(v.Namespace) + "/" + boldFgCyan.Sprintf(v.Kind) + "/" + boldFgCyan.Sprintf(v.Resource)
-		} else if v.Namespace != "" {
-			res.Resource = boldFgCyan.Sprintf(v.Namespace) + "/" + boldFgCyan.Sprintf(v.Kind) + "/" + boldFgCyan.Sprintf(v.Resource)
-			resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", v.Policy, ruleNameInResultKey, v.Namespace, v.Kind, v.Resource)
-		}
+				resultKey := fmt.Sprintf("%s-%s-%s-%s", v.Policy, ruleNameInResultKey, v.Kind, resource)
+				found, _ := isNamespacedPolicy(v.Policy)
+				var ns string
+				ns, v.Policy = getUserDefinedPolicyNameAndNamespace(v.Policy)
+				if found && v.Namespace != "" {
+					resultKey = fmt.Sprintf("%s-%s-%s-%s-%s-%s", ns, v.Policy, ruleNameInResultKey, v.Namespace, v.Kind, resource)
+				} else if found {
+					resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", ns, v.Policy, ruleNameInResultKey, v.Kind, resource)
+					res.Policy = boldFgCyan.Sprintf(ns) + "/" + boldFgCyan.Sprintf(v.Policy)
+					res.Resource = boldFgCyan.Sprintf(v.Namespace) + "/" + boldFgCyan.Sprintf(v.Kind) + "/" + boldFgCyan.Sprintf(resource)
+				} else if v.Namespace != "" {
+					res.Resource = boldFgCyan.Sprintf(v.Namespace) + "/" + boldFgCyan.Sprintf(v.Kind) + "/" + boldFgCyan.Sprintf(resource)
+					resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", v.Policy, ruleNameInResultKey, v.Namespace, v.Kind, resource)
+				}
 
-		var testRes policyreportv1alpha2.PolicyReportResult
-		if val, ok := resps[resultKey]; ok {
-			testRes = val
-		} else {
-			log.Log.V(2).Info("result not found", "key", resultKey)
-			res.Result = boldYellow.Sprintf("Not found")
-			rc.Fail++
-			table = append(table, res)
-			ftable = append(ftable, res)
-			continue
-		}
+				var testRes policyreportv1alpha2.PolicyReportResult
+				if val, ok := resps[resultKey]; ok {
+					testRes = val
+				} else {
+					log.Log.V(2).Info("result not found", "key", resultKey)
+					res.Result = boldYellow.Sprintf("Not found")
+					rc.Fail++
+					table = append(table, *res)
+					ftable = append(ftable, *res)
+					continue
+				}
 
-		if v.Result == "" && v.Status != "" {
-			v.Result = v.Status
-		}
+				if v.Result == "" && v.Status != "" {
+					v.Result = v.Status
+				}
 
-		if testRes.Result == v.Result {
-			res.Result = boldGreen.Sprintf("Pass")
-			if testRes.Result == policyreportv1alpha2.StatusSkip {
-				res.Result = boldGreen.Sprintf("Pass")
-				rc.Skip++
-			} else {
-				res.Result = boldGreen.Sprintf("Pass")
-				rc.Pass++
+				if testRes.Result == v.Result {
+					res.Result = boldGreen.Sprintf("Pass")
+					if testRes.Result == policyreportv1alpha2.StatusSkip {
+						res.Result = boldGreen.Sprintf("Pass")
+						rc.Skip++
+					} else {
+						res.Result = boldGreen.Sprintf("Pass")
+						rc.Pass++
+					}
+				} else {
+					log.Log.V(2).Info("result mismatch", "expected", v.Result, "received", testRes.Result, "key", resultKey)
+					res.Result = boldRed.Sprintf("Fail")
+					rc.Fail++
+					ftable = append(ftable, *res)
+				}
+
+				table = append(table, *res)
+			}
+		} else if v.Resource != "" {
+			countDeprecatedResource++
+			res.Resource = boldFgCyan.Sprintf(v.Namespace) + "/" + boldFgCyan.Sprintf(v.Kind) + "/" + boldFgCyan.Sprintf(v.Resource)
+			var ruleNameInResultKey string
+			if v.AutoGeneratedRule != "" {
+				ruleNameInResultKey = fmt.Sprintf("%s-%s", v.AutoGeneratedRule, v.Rule)
+			} else {
+				ruleNameInResultKey = v.Rule
 			}
-		} else {
-			log.Log.V(2).Info("result mismatch", "expected", v.Result, "received", testRes.Result, "key", resultKey)
-			res.Result = boldRed.Sprintf("Fail")
-			rc.Fail++
-			ftable = append(ftable, res)
-		}
 
-		table = append(table, res)
+			resultKey := fmt.Sprintf("%s-%s-%s-%s", v.Policy, ruleNameInResultKey, v.Kind, v.Resource)
+			found, _ := isNamespacedPolicy(v.Policy)
+			var ns string
+			ns, v.Policy = getUserDefinedPolicyNameAndNamespace(v.Policy)
+			if found && v.Namespace != "" {
+				resultKey = fmt.Sprintf("%s-%s-%s-%s-%s-%s", ns, v.Policy, ruleNameInResultKey, v.Namespace, v.Kind, v.Resource)
+			} else if found {
+				resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", ns, v.Policy, ruleNameInResultKey, v.Kind, v.Resource)
+				res.Policy = boldFgCyan.Sprintf(ns) + "/" + boldFgCyan.Sprintf(v.Policy)
+				res.Resource = boldFgCyan.Sprintf(v.Namespace) + "/" + boldFgCyan.Sprintf(v.Kind) + "/" + boldFgCyan.Sprintf(v.Resource)
+			} else if v.Namespace != "" {
+				res.Resource = boldFgCyan.Sprintf(v.Namespace) + "/" + boldFgCyan.Sprintf(v.Kind) + "/" + boldFgCyan.Sprintf(v.Resource)
+				resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", v.Policy, ruleNameInResultKey, v.Namespace, v.Kind, v.Resource)
+			}
+
+			var testRes policyreportv1alpha2.PolicyReportResult
+			if val, ok := resps[resultKey]; ok {
+				testRes = val
+			} else {
+				log.Log.V(2).Info("result not found", "key", resultKey)
+				res.Result = boldYellow.Sprintf("Not found")
+				rc.Fail++
+				table = append(table, *res)
+				ftable = append(ftable, *res)
+				continue
+			}
+
+			if v.Result == "" && v.Status != "" {
+				v.Result = v.Status
+			}
+
+			if testRes.Result == v.Result {
+				res.Result = boldGreen.Sprintf("Pass")
+				if testRes.Result == policyreportv1alpha2.StatusSkip {
+					res.Result = boldGreen.Sprintf("Pass")
+					rc.Skip++
+				} else {
+					res.Result = boldGreen.Sprintf("Pass")
+					rc.Pass++
+				}
+			} else {
+				log.Log.V(2).Info("result mismatch", "expected", v.Result, "received", testRes.Result, "key", resultKey)
+				res.Result = boldRed.Sprintf("Fail")
+				rc.Fail++
+				ftable = append(ftable, *res)
+			}
+
+			table = append(table, *res)
+		}
 	}
 
+	if countDeprecatedResource > 0 {
+		fmt.Printf("\n Note : The resource field is being deprecated in 1.8.0 release. Please provide the resources under the resources parameter as an array in the results field \n")
+	}
 	printer.BorderTop, printer.BorderBottom, printer.BorderLeft, printer.BorderRight = true, true, true, true
 	printer.CenterSeparator = "│"
 	printer.ColumnSeparator = "│"

test/cli/test/resource_lists/kyverno-test.yaml

@@ -0,0 +1,19 @@
+name: resource-lists
+policies:
+  -  policy.yaml
+resources:
+  -  resource.yaml
+results:
+  - policy: resource-lists
+    rule: require-image-tag
+    resources: 
+    - myapp-pod1
+    - myapp-pod2
+    kind: Pod
+    result: pass
+  - policy: resource-lists
+    rule: validate-image-tag
+    resources: 
+    - myapp-pod3
+    kind: Pod
+    result: pass 

test/cli/test/resource_lists/policy.yaml

@@ -0,0 +1,35 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: resource-lists
+  annotations:
+    policies.kyverno.io/category: Best Practices
+    policies.kyverno.io/description: >-
+      The ':latest' tag is mutable and can lead to unexpected errors if the 
+      image changes. A best practice is to use an immutable tag that maps to 
+      a specific version of an application pod.      
+spec:
+  validationFailureAction: audit
+  rules:
+  - name: require-image-tag
+    match:
+      resources:
+        kinds:
+        - Pod
+    validate:
+      message: "An image tag is required."  
+      pattern:
+        spec:
+          containers:
+          - image: "*:*"
+  - name: validate-image-tag
+    match:
+      resources:
+        kinds:
+        - Pod
+    validate:
+      message: "Using a mutable image tag e.g. 'latest' is not allowed."
+      pattern:
+        spec:
+          containers:
+          - image: "!*:latest" 

test/cli/test/resource_lists/resource.yaml

@@ -0,0 +1,34 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: myapp-pod1
+  labels:
+    app: myapp1
+spec: 
+  containers:
+  - name: nginx
+    image: nginx:1.12
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: myapp-pod2
+  labels:
+    app: myapp2
+spec: 
+  containers:
+  - name: nginx
+    image: nginx:1.12
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: myapp-pod3
+  labels:
+    app: myapp3
+spec: 
+  containers:
+  - name: nginx
+    image: ngnix:1.12