fix: multiple test cases for generate policy lead to wrong test results (#8197)

* fix: multiple test cases for generate policy lead to wrong test results

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* add cli test

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

cmd/cli/kubectl-kyverno/test/test.go

@@ -397,6 +397,8 @@ func buildPolicyResults(
 							if _, ok := results[resultsKey]; !ok {
 								results[resultsKey] = result
 							}
+
+							buildPolicyResultsForGenerate(resp, test, policyNamespace, policyName, resourceNamespace, resourceKind, resourceName, results, isGit, policyResourcePath, fs)
 						}
 					}
 				}
@@ -433,39 +435,7 @@ func buildPolicyResults(
 					if _, ok := results[resultsKey]; !ok {
 						results[resultsKey] = result
 					}
-				}
-			}
-
-			for _, rule := range resp.PolicyResponse.Rules {
-				if rule.RuleType() != engineapi.Generation || test.Rule != rule.Name() {
-					continue
-				}
-
-				var resultsKey []string
-				var resultKey string
-				var result policyreportv1alpha2.PolicyReportResult
-				resultsKey = GetAllPossibleResultsKey(policyNamespace, policyName, rule.Name(), resourceNamespace, resourceKind, resourceName, test.IsValidatingAdmissionPolicy)
-				for _, key := range resultsKey {
-					if val, ok := results[key]; ok {
-						result = val
-						resultKey = key
-					} else {
-						continue
-					}
-
-					if rule.Status() == engineapi.RuleStatusSkip {
-						result.Result = policyreportv1alpha2.StatusSkip
-					} else if rule.Status() == engineapi.RuleStatusError {
-						result.Result = policyreportv1alpha2.StatusError
-					} else {
-						var x string
-						result.Result = policyreportv1alpha2.StatusFail
-						x = getAndCompareResource(test.GeneratedResource, rule.GeneratedResource(), isGit, policyResourcePath, fs, true)
-						if x == "pass" {
-							result.Result = policyreportv1alpha2.StatusPass
-						}
-					}
-					results[resultKey] = result
+					buildPolicyResultsForGenerate(resp, test, policyNamespace, policyName, resourceNamespace, resourceKind, resourceName, results, isGit, policyResourcePath, fs)
 				}
 			}
 
@@ -549,6 +519,41 @@ func buildPolicyResults(
 	return results, testResults
 }
 
+func buildPolicyResultsForGenerate(resp engineapi.EngineResponse, test api.TestResults, policyNamespace string, policyName string, resourceNamespace string, resourceKind string, resourceName string, results map[string]policyreportv1alpha2.PolicyReportResult, isGit bool, policyResourcePath string, fs billy.Filesystem) {
+	for _, rule := range resp.PolicyResponse.Rules {
+		if rule.RuleType() != engineapi.Generation || test.Rule != rule.Name() {
+			continue
+		}
+
+		var resultsKey []string
+		var resultKey string
+		var result policyreportv1alpha2.PolicyReportResult
+		resultsKey = GetAllPossibleResultsKey(policyNamespace, policyName, rule.Name(), resourceNamespace, resourceKind, resourceName, test.IsValidatingAdmissionPolicy)
+		for _, key := range resultsKey {
+			if val, ok := results[key]; ok {
+				result = val
+				resultKey = key
+			} else {
+				continue
+			}
+
+			if rule.Status() == engineapi.RuleStatusSkip {
+				result.Result = policyreportv1alpha2.StatusSkip
+			} else if rule.Status() == engineapi.RuleStatusError {
+				result.Result = policyreportv1alpha2.StatusError
+			} else {
+				var x string
+				result.Result = policyreportv1alpha2.StatusFail
+				x = getAndCompareResource(test.GeneratedResource, rule.GeneratedResource(), isGit, policyResourcePath, fs, true)
+				if x == "pass" {
+					result.Result = policyreportv1alpha2.StatusPass
+				}
+			}
+			results[resultKey] = result
+		}
+	}
+}
+
 func GetAllPossibleResultsKey(policyNamespace, policy, rule, resourceNamespace, kind, resource string, isVAP bool) []string {
 	var resultsKey []string
 	var resultKey1, resultKey2, resultKey3, resultKey4 string

test/cli/test-generate/multiple-resources/generated-resource-1.yaml

@@ -0,0 +1,5 @@
+apiVersion: foo/v1
+kind: Foo
+metadata:
+  name: foo-resource-a
+  namespace: default

test/cli/test-generate/multiple-resources/generated-resource-2.yaml

@@ -0,0 +1,5 @@
+apiVersion: foo/v1
+kind: Foo
+metadata:
+  name: foo-resource-b
+  namespace: default

test/cli/test-generate/multiple-resources/kyverno-test.yaml

@@ -0,0 +1,18 @@
+name: multiple-resources
+policies:
+  - policy.yaml
+resources:
+  - resources.yaml
+results:
+  - policy: test-policy
+    rule: rule
+    resource: resource-a
+    generatedResource: generated-resource-1.yaml
+    kind: Deployment
+    result: pass
+  - policy: test-policy
+    rule: rule
+    resource: resource-b
+    generatedResource: generated-resource-2.yaml
+    kind: Deployment
+    result: pass

test/cli/test-generate/multiple-resources/policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: test-policy
+spec:
+  rules:
+    - name: rule
+      match:
+        any:
+          - resources:
+              kinds:
+                - Deployment
+      generate:
+        apiVersion: foo/v1
+        kind: Foo
+        name: "foo-{{request.object.metadata.name}}"
+        namespace: "{{request.object.metadata.namespace}}"

test/cli/test-generate/multiple-resources/resources.yaml

@@ -0,0 +1,11 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: resource-a
+  namespace: default
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: resource-b
+  namespace: default