refactor: cli test command test execution (#8266)

* refactor: cli test command test execution

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* error

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* tests

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

cmd/cli/kubectl-kyverno/commands/apply/command.go

@@ -140,13 +140,13 @@ func (c *ApplyCommandConfig) applyCommandHelper() (*common.ResultCounts, []*unst
 	}
 	var userInfo v1beta1.RequestInfo
 	if c.UserInfoPath != "" {
-		userInfo, err = common.GetUserInfoFromPath(nil, c.UserInfoPath, false, "")
+		userInfo, err = common.GetUserInfoFromPath(nil, c.UserInfoPath, "")
 		if err != nil {
 			fmt.Printf("Error: failed to load request info\nCause: %s\n", err)
 			osExit(1)
 		}
 	}
-	variables, globalValMap, valuesMap, namespaceSelectorMap, subresources, err := common.GetVariable(c.Variables, nil, c.ValuesFile, nil, false, "")
+	variables, globalValMap, valuesMap, namespaceSelectorMap, subresources, err := common.GetVariable(c.Variables, nil, c.ValuesFile, nil, "")
 	if err != nil {
 		if !sanitizederror.IsErrorSanitized(err) {
 			return nil, nil, skipInvalidPolicies, nil, sanitizederror.NewWithError("failed to decode yaml", err)

cmd/cli/kubectl-kyverno/commands/test/command.go

@@ -2,11 +2,9 @@ package test
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 
 	"github.com/go-git/go-billy/v5"
-	policyreportv1alpha2 "github.com/kyverno/kyverno/api/policyreport/v1alpha2"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/output/color"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/output/table"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test/api"
@@ -109,30 +107,36 @@ func testCommandExecute(
 	}
 	if len(tests) == 0 {
 		if len(errors) == 0 {
-			os.Exit(0)
+			return nil
 		} else {
-			os.Exit(1)
+			// TODO aggregate errors
+			return errors[0]
 		}
 	}
 	rc := &resultCounts{}
 	var table table.Table
 	for _, test := range tests {
 		if test.Err == nil {
-			resourcePath := filepath.Dir(test.Path)
-			if tests, responses, err := applyPoliciesFromPath(
-				test,
-				resourcePath,
-				rc,
-				openApiManager,
-				filter,
-				false,
-			); err != nil {
-				return sanitizederror.NewWithError("failed to apply test command", err)
-			} else if t, err := printTestResult(tests, responses, rc, failOnly, detailedResults, test.Fs, resourcePath); err != nil {
-				return sanitizederror.NewWithError("failed to print test result:", err)
-			} else {
-				table.AddFailed(t.RawRows...)
+			// filter results
+			var filteredResults []api.TestResults
+			for _, res := range test.Test.Results {
+				if filter.Apply(res) {
+					filteredResults = append(filteredResults, res)
+				}
 			}
+			if len(filteredResults) == 0 {
+				continue
+			}
+			resourcePath := filepath.Dir(test.Path)
+			responses, err := runTest(openApiManager, test, false)
+			if err != nil {
+				return sanitizederror.NewWithError("failed to run test", err)
+			}
+			t, err := printTestResult(filteredResults, responses, rc, failOnly, detailedResults, test.Fs, resourcePath)
+			if err != nil {
+				return sanitizederror.NewWithError("failed to print test result:", err)
+			}
+			table.AddFailed(t.RawRows...)
 		}
 	}
 	if !failOnly {
@@ -145,7 +149,7 @@ func testCommandExecute(
 		if !failOnly {
 			printFailedTestResult(table, detailedResults)
 		}
-		os.Exit(1)
+		return fmt.Errorf("%d tests failed", rc.Fail)
 	}
 	return nil
 }
@@ -220,102 +224,3 @@ func lookupRuleResponses(test api.TestResults, responses ...engineapi.RuleRespon
 	}
 	return matches
 }
-
-func printTestResult(
-	tests []api.TestResults,
-	responses []engineapi.EngineResponse,
-	rc *resultCounts,
-	failOnly bool,
-	detailedResults bool,
-	fs billy.Filesystem,
-	resoucePath string,
-) (table.Table, error) {
-	printer := table.NewTablePrinter()
-	var resultsTable table.Table
-	var countDeprecatedResource int
-	testCount := 1
-	for _, test := range tests {
-		// lookup matching engine responses (without the resource name)
-		// to reduce the search scope
-		responses := lookupEngineResponses(test, "", responses...)
-		// TODO fix deprecated fields
-		// identify the resources to be looked up
-		var resources []string
-		if test.Resources != nil {
-			resources = append(resources, test.Resources...)
-		} else if test.Resource != "" {
-			countDeprecatedResource++
-			resources = append(resources, test.Resource)
-		}
-		for _, resource := range resources {
-			var rows []table.Row
-			// lookup matching engine responses (with the resource name this time)
-			for _, response := range lookupEngineResponses(test, resource, responses...) {
-				// lookup matching rule responses
-				for _, rule := range lookupRuleResponses(test, response.PolicyResponse.Rules...) {
-					// perform test checks
-					ok, message, reason := checkResult(test, fs, resoucePath, response, rule)
-					// if checks failed but we were expecting a fail it's considered a success
-					success := ok || (!ok && test.Result == policyreportv1alpha2.StatusFail)
-					row := table.Row{
-						CompactRow: table.CompactRow{
-							ID:        testCount,
-							Policy:    color.Policy("", test.Policy),
-							Rule:      color.Rule(test.Rule),
-							Resource:  color.Resource(test.Kind, test.Namespace, resource),
-							Reason:    reason,
-							IsFailure: !success,
-						},
-						Message: message,
-					}
-					if success {
-						row.Result = color.ResultPass()
-						if test.Result == policyreportv1alpha2.StatusSkip {
-							rc.Skip++
-						} else {
-							rc.Pass++
-						}
-					} else {
-						row.Result = color.ResultFail()
-						rc.Fail++
-					}
-					testCount++
-					rows = append(rows, row)
-				}
-			}
-			// if not found
-			if len(rows) == 0 {
-				row := table.Row{
-					CompactRow: table.CompactRow{
-						ID:        testCount,
-						Policy:    color.Policy("", test.Policy),
-						Rule:      color.Rule(test.Rule),
-						Resource:  color.Resource(test.Kind, test.Namespace, resource),
-						IsFailure: true,
-						Result:    color.ResultFail(),
-						Reason:    color.NotFound(),
-					},
-					Message: color.NotFound(),
-				}
-				testCount++
-				resultsTable.Add(row)
-				rc.Fail++
-			} else {
-				resultsTable.Add(rows...)
-			}
-		}
-	}
-	fmt.Printf("\n")
-	printer.Print(resultsTable.Rows(detailedResults))
-	return resultsTable, nil
-}
-
-func printFailedTestResult(resultsTable table.Table, detailedResults bool) {
-	printer := table.NewTablePrinter()
-	for i := range resultsTable.RawRows {
-		resultsTable.RawRows[i].ID = i + 1
-	}
-	fmt.Printf("Aggregated Failed Test Cases : ")
-	fmt.Println()
-	printer.Print(resultsTable.Rows(detailedResults))
-}

cmd/cli/kubectl-kyverno/commands/test/command_test.go

@@ -1,93 +0,0 @@
-package test
-
-import (
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/go-git/go-billy/v5/memfs"
-	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test/api"
-	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/common"
-	"gotest.tools/assert"
-	"sigs.k8s.io/yaml"
-)
-
-func Test_selectResourcesForCheck(t *testing.T) {
-
-	type TestCase struct {
-		testFile           string
-		expectedResources  int
-		expectedDuplicates int
-		expectedUnused     int
-	}
-	baseTestDir := "../../../../../test/cli/test-unit/selectResourcesForCheck/"
-	testcases := []*TestCase{
-		{
-
-			testFile:           "kyverno-test-duplicated-with-resource.yaml",
-			expectedResources:  3,
-			expectedDuplicates: 1,
-			expectedUnused:     3,
-		},
-		{
-			testFile:           "kyverno-test-duplicated-with-resources.yaml",
-			expectedResources:  3,
-			expectedDuplicates: 1,
-			expectedUnused:     3,
-		},
-		{
-			testFile:           "kyverno-test-uniq-with-resource.yaml",
-			expectedResources:  3,
-			expectedDuplicates: 0,
-			expectedUnused:     3,
-		},
-		{
-			testFile:           "kyverno-test-uniq-with-resources.yaml",
-			expectedResources:  3,
-			expectedDuplicates: 0,
-			expectedUnused:     3,
-		},
-	}
-	fs := memfs.New()
-	for _, tc := range testcases {
-
-		// read test spec
-		values := &api.Test{}
-		testBytes, err := os.ReadFile(filepath.Join(baseTestDir, tc.testFile))
-		assert.NilError(t, err)
-		err = yaml.Unmarshal(testBytes, values)
-		assert.NilError(t, err)
-
-		// read policies
-		policies, validatingAdmissionPolicies, err := common.GetPoliciesFromPaths(
-			fs,
-			[]string{filepath.Join(baseTestDir, values.Policies[0])},
-			false,
-			filepath.Join(baseTestDir, values.Resources[0]),
-		)
-		assert.NilError(t, err)
-
-		// read resources
-		resources, err := common.GetResourceAccordingToResourcePath(
-			fs,
-			[]string{filepath.Join(baseTestDir, values.Resources[0])},
-			false,
-			policies,
-			validatingAdmissionPolicies,
-			nil,
-			"",
-			false,
-			false,
-			filepath.Join(baseTestDir, values.Policies[0]),
-		)
-		assert.NilError(t, err)
-
-		selected, duplicates, unused := selectResourcesForCheckInternal(resources, values)
-		assert.Equal(t, len(selected), tc.expectedResources,
-			"Did not get the expected number of resources for test %s", tc.testFile)
-		assert.Equal(t, duplicates, tc.expectedDuplicates,
-			"Did not get the expected number of duplicates for test %s", tc.testFile)
-		assert.Equal(t, unused, tc.expectedUnused,
-			"Did not get the expected number of unused resources for test %s", tc.testFile)
-	}
-}

cmd/cli/kubectl-kyverno/commands/test/output.go

@@ -0,0 +1,111 @@
+package test
+
+import (
+	"fmt"
+
+	"github.com/go-git/go-billy/v5"
+	policyreportv1alpha2 "github.com/kyverno/kyverno/api/policyreport/v1alpha2"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/output/color"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/output/table"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test/api"
+	engineapi "github.com/kyverno/kyverno/pkg/engine/api"
+)
+
+func printTestResult(
+	tests []api.TestResults,
+	responses []engineapi.EngineResponse,
+	rc *resultCounts,
+	failOnly bool,
+	detailedResults bool,
+	fs billy.Filesystem,
+	resoucePath string,
+) (table.Table, error) {
+	printer := table.NewTablePrinter()
+	var resultsTable table.Table
+	var countDeprecatedResource int
+	testCount := 1
+	for _, test := range tests {
+		// lookup matching engine responses (without the resource name)
+		// to reduce the search scope
+		responses := lookupEngineResponses(test, "", responses...)
+		// TODO fix deprecated fields
+		// identify the resources to be looked up
+		var resources []string
+		if test.Resources != nil {
+			resources = append(resources, test.Resources...)
+		} else if test.Resource != "" {
+			countDeprecatedResource++
+			resources = append(resources, test.Resource)
+		}
+		for _, resource := range resources {
+			var rows []table.Row
+			// lookup matching engine responses (with the resource name this time)
+			for _, response := range lookupEngineResponses(test, resource, responses...) {
+				// lookup matching rule responses
+				for _, rule := range lookupRuleResponses(test, response.PolicyResponse.Rules...) {
+					// perform test checks
+					ok, message, reason := checkResult(test, fs, resoucePath, response, rule)
+					// if checks failed but we were expecting a fail it's considered a success
+					success := ok || (!ok && test.Result == policyreportv1alpha2.StatusFail)
+					row := table.Row{
+						CompactRow: table.CompactRow{
+							ID:        testCount,
+							Policy:    color.Policy("", test.Policy),
+							Rule:      color.Rule(test.Rule),
+							Resource:  color.Resource(test.Kind, test.Namespace, resource),
+							Reason:    reason,
+							IsFailure: !success,
+						},
+						Message: message,
+					}
+					if success {
+						row.Result = color.ResultPass()
+						if test.Result == policyreportv1alpha2.StatusSkip {
+							rc.Skip++
+						} else {
+							rc.Pass++
+						}
+					} else {
+						row.Result = color.ResultFail()
+						rc.Fail++
+					}
+					testCount++
+					rows = append(rows, row)
+				}
+			}
+			// if not found
+			if len(rows) == 0 {
+				row := table.Row{
+					CompactRow: table.CompactRow{
+						ID:        testCount,
+						Policy:    color.Policy("", test.Policy),
+						Rule:      color.Rule(test.Rule),
+						Resource:  color.Resource(test.Kind, test.Namespace, resource),
+						IsFailure: true,
+						Result:    color.ResultFail(),
+						Reason:    color.NotFound(),
+					},
+					Message: color.NotFound(),
+				}
+				testCount++
+				resultsTable.Add(row)
+				rc.Fail++
+			} else {
+				resultsTable.Add(rows...)
+			}
+		}
+	}
+	fmt.Printf("\n")
+	printer.Print(resultsTable.Rows(detailedResults))
+	return resultsTable, nil
+}
+
+func printFailedTestResult(resultsTable table.Table, detailedResults bool) {
+	printer := table.NewTablePrinter()
+	for i := range resultsTable.RawRows {
+		resultsTable.RawRows[i].ID = i + 1
+	}
+	fmt.Printf("Aggregated Failed Test Cases : ")
+	fmt.Println()
+	printer.Print(resultsTable.Rows(detailedResults))
+}

cmd/cli/kubectl-kyverno/commands/test/test.go

@@ -2,13 +2,11 @@ package test
 
 import (
 	"fmt"
-	"os"
 
-	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
 	"github.com/kyverno/kyverno/api/kyverno/v1beta1"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/output/pluralize"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/resource"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test"
-	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test/api"
-	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test/filter"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/common"
 	pathutils "github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/path"
 	sanitizederror "github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/sanitizedError"
@@ -20,123 +18,89 @@ import (
 	engineapi "github.com/kyverno/kyverno/pkg/engine/api"
 	"github.com/kyverno/kyverno/pkg/openapi"
 	policyvalidation "github.com/kyverno/kyverno/pkg/validation/policy"
-	"k8s.io/api/admissionregistration/v1alpha1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
-func applyPoliciesFromPath(
-	testCase test.TestCase,
-	policyResourcePath string,
-	rc *resultCounts,
-	openApiManager openapi.Manager,
-	filter filter.Filter,
-	auditWarn bool,
-) ([]api.TestResults, []engineapi.EngineResponse, error) {
-	var engineResponses []engineapi.EngineResponse
-	test := testCase.Test
-	fs := testCase.Fs
-	isGit := fs != nil
-	var dClient dclient.Interface
-	var resultCounts common.ResultCounts
-
+func runTest(openApiManager openapi.Manager, testCase test.TestCase, auditWarn bool) ([]engineapi.EngineResponse, error) {
+	// don't process test case with errors
+	if testCase.Err != nil {
+		return nil, testCase.Err
+	}
+	fmt.Println("Loading test", testCase.Path, "...")
 	store.SetLocal(true)
-
-	var filteredResults []api.TestResults
-	for _, res := range test.Results {
-		if filter.Apply(res) {
-			filteredResults = append(filteredResults, res)
-		}
-	}
-	test.Results = filteredResults
-
-	if len(test.Results) == 0 {
-		return nil, nil, nil
-	}
-
-	fmt.Printf("\nExecuting %s...\n", test.Name)
-	valuesFile := test.Variables
-	userInfoFile := test.UserInfo
-
-	variables, globalValMap, valuesMap, namespaceSelectorMap, subresources, err := common.GetVariable(nil, test.Values, test.Variables, fs, isGit, policyResourcePath)
+	isGit := testCase.Fs != nil
+	testDir := testCase.Dir()
+	var dClient dclient.Interface
+	// values/variables
+	fmt.Println("  Loading values/variables", "...")
+	variables, globalValMap, valuesMap, namespaceSelectorMap, subresources, err := common.GetVariable(
+		nil,
+		testCase.Test.Values,
+		testCase.Test.Variables,
+		testCase.Fs,
+		testDir,
+	)
 	if err != nil {
 		if !sanitizederror.IsErrorSanitized(err) {
-			return nil, nil, sanitizederror.NewWithError("failed to decode yaml", err)
+			err = sanitizederror.NewWithError("failed to decode yaml", err)
 		}
-		return nil, nil, err
+		return nil, err
 	}
-
-	// get the user info as request info from a different file
+	// user info
 	var userInfo v1beta1.RequestInfo
-
-	if userInfoFile != "" {
-		userInfo, err = common.GetUserInfoFromPath(fs, userInfoFile, isGit, policyResourcePath)
+	if testCase.Test.UserInfo != "" {
+		fmt.Println("  Loading user infos", "...")
+		userInfo, err = common.GetUserInfoFromPath(testCase.Fs, testCase.Test.UserInfo, testDir)
 		if err != nil {
-			fmt.Printf("Error: failed to load request info\nCause: %s\n", err)
-			os.Exit(1)
+			return nil, fmt.Errorf("Error: failed to load request info (%s)", err)
 		}
 	}
-
-	policyFullPath := pathutils.GetFullPaths(test.Policies, policyResourcePath, isGit)
-	resourceFullPath := pathutils.GetFullPaths(test.Resources, policyResourcePath, isGit)
-
-	policies, validatingAdmissionPolicies, err := common.GetPoliciesFromPaths(fs, policyFullPath, isGit, policyResourcePath)
+	// policies
+	fmt.Println("  Loading policies", "...")
+	policyFullPath := pathutils.GetFullPaths(testCase.Test.Policies, testDir, isGit)
+	policies, validatingAdmissionPolicies, err := common.GetPoliciesFromPaths(testCase.Fs, policyFullPath, isGit, testDir)
 	if err != nil {
-		fmt.Printf("Error: failed to load policies\nCause: %s\n", err)
-		os.Exit(1)
+		return nil, fmt.Errorf("Error: failed to load policies (%s)", err)
 	}
-
-	var filteredPolicies []kyvernov1.PolicyInterface
-	for _, p := range policies {
-		for _, res := range test.Results {
-			if p.GetName() == res.Policy {
-				filteredPolicies = append(filteredPolicies, p)
-				break
-			}
+	// resources
+	fmt.Println("  Loading resources", "...")
+	resourceFullPath := pathutils.GetFullPaths(testCase.Test.Resources, testDir, isGit)
+	resources, err := common.GetResourceAccordingToResourcePath(testCase.Fs, resourceFullPath, false, policies, validatingAdmissionPolicies, dClient, "", false, isGit, testDir)
+	if err != nil {
+		return nil, fmt.Errorf("Error: failed to load resources (%s)", err)
+	}
+	uniques, duplicates := resource.RemoveDuplicates(resources)
+	if len(duplicates) > 0 {
+		for dup := range duplicates {
+			fmt.Println("  Warning: found duplicated resource", dup.Kind, dup.Name, dup.Namespace)
 		}
 	}
-
-	var filteredVAPs []v1alpha1.ValidatingAdmissionPolicy
-	for _, p := range validatingAdmissionPolicies {
-		for _, res := range test.Results {
-			if p.GetName() == res.Policy {
-				filteredVAPs = append(filteredVAPs, p)
-				break
-			}
-		}
-	}
-	validatingAdmissionPolicies = filteredVAPs
-
+	// TODO document the code below
 	ruleToCloneSourceResource := map[string]string{}
-	for _, p := range filteredPolicies {
-		var filteredRules []kyvernov1.Rule
-
-		for _, rule := range autogen.ComputeRules(p) {
-			for _, res := range test.Results {
+	for _, policy := range policies {
+		for _, rule := range autogen.ComputeRules(policy) {
+			for _, res := range testCase.Test.Results {
 				if res.IsValidatingAdmissionPolicy {
 					continue
 				}
-
 				if rule.Name == res.Rule {
-					filteredRules = append(filteredRules, rule)
 					if rule.HasGenerate() {
 						ruleUnstr, err := generate.GetUnstrRule(rule.Generation.DeepCopy())
 						if err != nil {
 							fmt.Printf("Error: failed to get unstructured rule\nCause: %s\n", err)
 							break
 						}
-
 						genClone, _, err := unstructured.NestedMap(ruleUnstr.Object, "clone")
 						if err != nil {
 							fmt.Printf("Error: failed to read data\nCause: %s\n", err)
 							break
 						}
-
 						if len(genClone) != 0 {
 							if isGit {
 								ruleToCloneSourceResource[rule.Name] = res.CloneSourceResource
 							} else {
-								ruleToCloneSourceResource[rule.Name] = pathutils.GetFullPath(res.CloneSourceResource, policyResourcePath)
+								ruleToCloneSourceResource[rule.Name] = pathutils.GetFullPath(res.CloneSourceResource, testDir)
 							}
 						}
 					}
@@ -144,33 +108,14 @@ func applyPoliciesFromPath(
 				}
 			}
 		}
-		p.GetSpec().SetRules(filteredRules)
 	}
-	policies = filteredPolicies
-
-	resources, err := common.GetResourceAccordingToResourcePath(fs, resourceFullPath, false, policies, validatingAdmissionPolicies, dClient, "", false, isGit, policyResourcePath)
-	if err != nil {
-		fmt.Printf("Error: failed to load resources\nCause: %s\n", err)
-		os.Exit(1)
-	}
-
-	checkableResources := selectResourcesForCheck(resources, test)
-
-	msgPolicies := "1 policy"
-	if len(policies)+len(validatingAdmissionPolicies) > 1 {
-		msgPolicies = fmt.Sprintf("%d policies", len(policies)+len(validatingAdmissionPolicies))
-	}
-
-	msgResources := "1 resource"
-	if len(checkableResources) > 1 {
-		msgResources = fmt.Sprintf("%d resources", len(checkableResources))
-	}
-
-	if len(policies) > 0 && len(checkableResources) > 0 {
-		fmt.Printf("applying %s to %s... \n", msgPolicies, msgResources)
-	}
-
+	// execute engine
+	fmt.Println("  Applying", len(policies), pluralize.Pluralize(len(policies), "policy", "policies"), "to", len(uniques), pluralize.Pluralize(len(uniques), "resource", "resources"), "...")
+	var engineResponses []engineapi.EngineResponse
+	var resultCounts common.ResultCounts
+	// TODO loop through resources first, then through policies second
 	for _, policy := range policies {
+		// TODO we should return this info to the caller
 		_, err := policyvalidation.Validate(policy, nil, nil, true, openApiManager, config.KyvernoUserName(config.KyvernoServiceAccountName()))
 		if err != nil {
 			log.Log.Error(err, "skipping invalid policy", "name", policy.GetName())
@@ -180,21 +125,27 @@ func applyPoliciesFromPath(
 		matches := common.HasVariables(policy)
 		variable := common.RemoveDuplicateAndObjectVariables(matches)
 
-		if len(variable) > 0 {
-			if len(variables) == 0 {
-				// check policy in variable file
-				if valuesFile == "" || valuesMap[policy.GetName()] == nil {
-					fmt.Printf("test skipped for policy  %v  (as required variables are not provided by the users) \n \n", policy.GetName())
-				}
-			}
-		}
+		// TODO
+		// 	if len(variable) > 0 {
+		// 		if len(variables) == 0 {
+		// 			// check policy in variable file
+		// 			if valuesFile == "" || valuesMap[policy.GetName()] == nil {
+		// 				fmt.Printf("test skipped for policy  %v  (as required variables are not provided by the users) \n \n", policy.GetName())
+		// 			}
+		// 		}
+		// 	}
 
 		kindOnwhichPolicyIsApplied := common.GetKindsFromPolicy(policy, subresources, dClient)
 
-		for _, resource := range checkableResources {
+		for _, resource := range uniques {
 			thisPolicyResourceValues, err := common.CheckVariableForPolicy(valuesMap, globalValMap, policy.GetName(), resource.GetName(), resource.GetKind(), variables, kindOnwhichPolicyIsApplied, variable)
 			if err != nil {
-				return nil, nil, sanitizederror.NewWithError(fmt.Sprintf("policy `%s` have variables. pass the values for the variables for resource `%s` using set/values_file flag", policy.GetName(), resource.GetName()), err)
+				message := fmt.Sprintf(
+					"policy `%s` have variables. pass the values for the variables for resource `%s` using set/values_file flag",
+					policy.GetName(),
+					resource.GetName(),
+				)
+				return nil, sanitizederror.NewWithError(message, err)
 			}
 			applyPolicyConfig := common.ApplyPolicyConfig{
 				Policy:                    policy,
@@ -211,15 +162,15 @@ func applyPoliciesFromPath(
 			}
 			ers, err := common.ApplyPolicyOnResource(applyPolicyConfig)
 			if err != nil {
-				return nil, nil, sanitizederror.NewWithError(fmt.Errorf("failed to apply policy %v on resource %v", policy.GetName(), resource.GetName()).Error(), err)
+				message := fmt.Sprintf("failed to apply policy %v on resource %v", policy.GetName(), resource.GetName())
+				return nil, sanitizederror.NewWithError(message, err)
 			}
 			engineResponses = append(engineResponses, ers...)
 		}
 	}
-
 	validatingAdmissionPolicy := common.ValidatingAdmissionPolicies{}
 	for _, policy := range validatingAdmissionPolicies {
-		for _, resource := range resources {
+		for _, resource := range uniques {
 			applyPolicyConfig := common.ApplyPolicyConfig{
 				ValidatingAdmissionPolicy: policy,
 				Resource:                  resource,
@@ -230,62 +181,11 @@ func applyPoliciesFromPath(
 			}
 			ers, err := validatingAdmissionPolicy.ApplyPolicyOnResource(applyPolicyConfig)
 			if err != nil {
-				return nil, nil, sanitizederror.NewWithError(fmt.Errorf("failed to apply policy %v on resource %v", policy.GetName(), resource.GetName()).Error(), err)
+				message := fmt.Sprintf("failed to apply policy %v on resource %v", policy.GetName(), resource.GetName())
+				return nil, sanitizederror.NewWithError(message, err)
 			}
 			engineResponses = append(engineResponses, ers...)
 		}
 	}
-	return test.Results, engineResponses, nil
-}
-
-func selectResourcesForCheck(resources []*unstructured.Unstructured, values *api.Test) []*unstructured.Unstructured {
-	res, _, _ := selectResourcesForCheckInternal(resources, values)
-	return res
-}
-
-// selectResourcesForCheckInternal internal method to test duplicates and unused
-func selectResourcesForCheckInternal(resources []*unstructured.Unstructured, values *api.Test) ([]*unstructured.Unstructured, int, int) {
-	var duplicates int
-	var unused int
-	uniqResources := make(map[string]*unstructured.Unstructured)
-
-	for i := range resources {
-		r := resources[i]
-		key := fmt.Sprintf("%s/%s/%s", r.GetKind(), r.GetName(), r.GetNamespace())
-		if _, ok := uniqResources[key]; ok {
-			fmt.Println("skipping duplicate resource, resource :", r)
-			duplicates++
-		} else {
-			uniqResources[key] = r
-		}
-	}
-
-	selectedResources := map[string]*unstructured.Unstructured{}
-	for key := range uniqResources {
-		r := uniqResources[key]
-		for _, res := range values.Results {
-			if res.Kind == r.GetKind() {
-				for _, testr := range res.Resources {
-					if r.GetName() == testr {
-						selectedResources[key] = r
-					}
-				}
-				if r.GetName() == res.Resource {
-					selectedResources[key] = r
-				}
-			}
-		}
-	}
-
-	var checkableResources []*unstructured.Unstructured
-
-	for key := range selectedResources {
-		checkableResources = append(checkableResources, selectedResources[key])
-		delete(uniqResources, key)
-	}
-	for _, r := range uniqResources {
-		fmt.Println("skipping unused resource, resource :", r)
-		unused++
-	}
-	return checkableResources, duplicates, unused
+	return engineResponses, nil
 }

cmd/cli/kubectl-kyverno/output/pluralize/pluralize.go

@@ -0,0 +1,8 @@
+package pluralize
+
+func Pluralize(number int, singular string, plural string) string {
+	if number == 1 {
+		return singular
+	}
+	return plural
+}

cmd/cli/kubectl-kyverno/output/pluralize/pluralize_test.go

@@ -0,0 +1,38 @@
+package pluralize
+
+import "testing"
+
+func TestPluralize(t *testing.T) {
+	tests := []struct {
+		name     string
+		number   int
+		singular string
+		plural   string
+		want     string
+	}{{
+		name:     "singular",
+		number:   1,
+		singular: "policy",
+		plural:   "policies",
+		want:     "policy",
+	}, {
+		name:     "plural",
+		number:   2,
+		singular: "policy",
+		plural:   "policies",
+		want:     "policies",
+	}, {
+		name:     "zero",
+		number:   0,
+		singular: "policy",
+		plural:   "policies",
+		want:     "policies",
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := Pluralize(tt.number, tt.singular, tt.plural); got != tt.want {
+				t.Errorf("Pluralize() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/cli/kubectl-kyverno/resource/duplicate.go

@@ -0,0 +1,34 @@
+package resource
+
+import (
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type ResourceKey struct {
+	schema.GroupKind
+	Namespace string
+	Name      string
+}
+
+type ResourceMap = map[ResourceKey]*unstructured.Unstructured
+
+func RemoveDuplicates(resources []*unstructured.Unstructured) (ResourceMap, ResourceMap) {
+	duplicates := ResourceMap{}
+	uniques := ResourceMap{}
+	for _, resource := range resources {
+		if resource != nil {
+			key := ResourceKey{
+				GroupKind: resource.GroupVersionKind().GroupKind(),
+				Namespace: resource.GetNamespace(),
+				Name:      resource.GetName(),
+			}
+			if uniques[key] == nil {
+				uniques[key] = resource
+			} else {
+				duplicates[key] = resource
+			}
+		}
+	}
+	return uniques, duplicates
+}

cmd/cli/kubectl-kyverno/resource/duplicate_test.go

@@ -0,0 +1,42 @@
+package resource
+
+import (
+	"path/filepath"
+	"testing"
+
+	"gotest.tools/assert"
+)
+
+func TestRemoveDuplicates(t *testing.T) {
+	type TestCase struct {
+		testFile           string
+		expectedResources  int
+		expectedDuplicates int
+	}
+	baseTestDir := "../testdata/resources"
+	tests := []*TestCase{
+		{
+
+			testFile:           "with-duplicate.yaml",
+			expectedResources:  6,
+			expectedDuplicates: 1,
+		},
+		{
+			testFile:           "all-unique.yaml",
+			expectedResources:  6,
+			expectedDuplicates: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.testFile, func(t *testing.T) {
+			fileBytes, err := GetFileBytes(filepath.Join(baseTestDir, tt.testFile))
+			assert.NilError(t, err)
+			resources, err := GetUnstructuredResources(fileBytes)
+			assert.NilError(t, err)
+
+			uniques, duplicates := RemoveDuplicates(resources)
+			assert.Equal(t, len(uniques), tt.expectedResources, "Did not get the expected number of resources for test %s", tt.testFile)
+			assert.Equal(t, len(duplicates), tt.expectedDuplicates, "Did not get the expected number of duplicates for test %s", tt.testFile)
+		})
+	}
+}

cmd/cli/kubectl-kyverno/test/load.go

@@ -10,25 +10,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/yaml"
 )
 
-type TestCase struct {
-	Path string
-	Fs   billy.Filesystem
-	Test *api.Test
-	Err  error
-}
-
-type TestCases []TestCase
-
-func (tc TestCases) Errors() []TestCase {
-	var errors []TestCase
-	for _, test := range tc {
-		if test.Err != nil {
-			errors = append(errors, test)
-		}
-	}
-	return errors
-}
-
 func LoadTests(dirPath string, fileName string) (TestCases, error) {
 	return loadLocalTest(filepath.Clean(dirPath), fileName)
 }

cmd/cli/kubectl-kyverno/test/load_test.go

@@ -9,65 +9,6 @@ import (
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test/api"
 )
 
-func TestTestCases_Errors(t *testing.T) {
-	tests := []struct {
-		name string
-		tc   TestCases
-		want []TestCase
-	}{{
-		name: "nil",
-		tc:   nil,
-		want: nil,
-	}, {
-		name: "empty",
-		tc:   []TestCase{},
-		want: nil,
-	}, {
-		name: "no error",
-		tc:   TestCases([]TestCase{{}}),
-		want: nil,
-	}, {
-		name: "one error",
-		tc: []TestCase{{
-			Err: errors.New("error 1"),
-		}},
-		want: []TestCase{{
-			Err: errors.New("error 1"),
-		}},
-	}, {
-		name: "two errors",
-		tc: []TestCase{{
-			Err: errors.New("error 1"),
-		}, {
-			Err: errors.New("error 2"),
-		}},
-		want: []TestCase{{
-			Err: errors.New("error 1"),
-		}, {
-			Err: errors.New("error 2"),
-		}},
-	}, {
-		name: "mixed",
-		tc: []TestCase{{
-			Err: errors.New("error 1"),
-		}, {}, {
-			Err: errors.New("error 2"),
-		}, {}},
-		want: []TestCase{{
-			Err: errors.New("error 1"),
-		}, {
-			Err: errors.New("error 2"),
-		}},
-	}}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := tt.tc.Errors(); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("TestCases.Errors() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
 func TestLoadTests(t *testing.T) {
 	tests := []struct {
 		name     string

cmd/cli/kubectl-kyverno/test/test_case.go

@@ -0,0 +1,19 @@
+package test
+
+import (
+	"path/filepath"
+
+	"github.com/go-git/go-billy/v5"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test/api"
+)
+
+type TestCase struct {
+	Path string
+	Fs   billy.Filesystem
+	Test *api.Test
+	Err  error
+}
+
+func (tc TestCase) Dir() string {
+	return filepath.Dir(tc.Path)
+}

cmd/cli/kubectl-kyverno/test/test_case_test.go

@@ -0,0 +1,37 @@
+package test
+
+import (
+	"testing"
+
+	"github.com/go-git/go-billy/v5"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test/api"
+)
+
+func TestTestCase_Dir(t *testing.T) {
+	type fields struct {
+		Path string
+		Fs   billy.Filesystem
+		Test *api.Test
+		Err  error
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   string
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tc := TestCase{
+				Path: tt.fields.Path,
+				Fs:   tt.fields.Fs,
+				Test: tt.fields.Test,
+				Err:  tt.fields.Err,
+			}
+			if got := tc.Dir(); got != tt.want {
+				t.Errorf("TestCase.Dir() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/cli/kubectl-kyverno/test/test_cases.go

@@ -0,0 +1,13 @@
+package test
+
+type TestCases []TestCase
+
+func (tc TestCases) Errors() []TestCase {
+	var errors []TestCase
+	for _, test := range tc {
+		if test.Err != nil {
+			errors = append(errors, test)
+		}
+	}
+	return errors
+}

cmd/cli/kubectl-kyverno/test/test_cases_test.go

@@ -0,0 +1,66 @@
+package test
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+)
+
+func TestTestCases_Errors(t *testing.T) {
+	tests := []struct {
+		name string
+		tc   TestCases
+		want []TestCase
+	}{{
+		name: "nil",
+		tc:   nil,
+		want: nil,
+	}, {
+		name: "empty",
+		tc:   []TestCase{},
+		want: nil,
+	}, {
+		name: "no error",
+		tc:   TestCases([]TestCase{{}}),
+		want: nil,
+	}, {
+		name: "one error",
+		tc: []TestCase{{
+			Err: errors.New("error 1"),
+		}},
+		want: []TestCase{{
+			Err: errors.New("error 1"),
+		}},
+	}, {
+		name: "two errors",
+		tc: []TestCase{{
+			Err: errors.New("error 1"),
+		}, {
+			Err: errors.New("error 2"),
+		}},
+		want: []TestCase{{
+			Err: errors.New("error 1"),
+		}, {
+			Err: errors.New("error 2"),
+		}},
+	}, {
+		name: "mixed",
+		tc: []TestCase{{
+			Err: errors.New("error 1"),
+		}, {}, {
+			Err: errors.New("error 2"),
+		}, {}},
+		want: []TestCase{{
+			Err: errors.New("error 1"),
+		}, {
+			Err: errors.New("error 2"),
+		}},
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.tc.Errors(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("TestCases.Errors() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/cli/kubectl-kyverno/test/test_result.go

@@ -0,0 +1,12 @@
+package test
+
+import (
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test/api"
+	engineapi "github.com/kyverno/kyverno/pkg/engine/api"
+)
+
+type TestResult struct {
+	EngineResponses []engineapi.EngineResponse
+	Results         []api.TestResults
+	Err             error
+}

cmd/cli/kubectl-kyverno/utils/common/common.go

@@ -584,9 +584,9 @@ func handleGeneratePolicy(generateResponse *engineapi.EngineResponse, policyCont
 }
 
 // GetUserInfoFromPath - get the request info as user info from a given path
-func GetUserInfoFromPath(fs billy.Filesystem, path string, isGit bool, policyResourcePath string) (kyvernov1beta1.RequestInfo, error) {
+func GetUserInfoFromPath(fs billy.Filesystem, path string, policyResourcePath string) (kyvernov1beta1.RequestInfo, error) {
 	userInfo := &kyvernov1beta1.RequestInfo{}
-	if isGit {
+	if fs != nil {
 		filep, err := fs.Open(filepath.Join(policyResourcePath, path))
 		if err != nil {
 			fmt.Printf("Unable to open userInfo file: %s. \nerror: %s", path, err)

cmd/cli/kubectl-kyverno/utils/common/variables.go

@@ -29,7 +29,6 @@ func GetVariable(
 	vals *api.Values,
 	valuesFile string,
 	fs billy.Filesystem,
-	isGit bool,
 	policyResourcePath string,
 ) (map[string]string, map[string]string, map[string]map[string]api.Resource, map[string]map[string]string, []api.Subresource, error) {
 	if vals == nil && valuesFile != "" {

test/cli/test-unit/selectResourcesForCheck/dummy-policy.yaml

@@ -1,8 +0,0 @@
-apiVersion: kyverno.io/v1
-kind: ClusterPolicy
-metadata:
-  name: just a dummy policy
-spec:
-  rules:
-    - name: dummy-rule-1
-    - name: dummy-rule-2

test/cli/test-unit/selectResourcesForCheck/kyverno-test-duplicated-with-resource.yaml

@@ -1,17 +0,0 @@
-name: dummy-policy
-policies:
-  - dummy-policy.yaml
-resources:
-  - resource-duplicates.yaml
-
-results:
-  - policy: dummy-policy
-    rule: require-image-tag
-    resource: myapp-pod1
-    kind: Pod
-    result: pass
-  - policy: dummy-policy
-    rule: require-image-tag
-    resource: myapp-pod2
-    kind: Pod
-    result: pass

test/cli/test-unit/selectResourcesForCheck/kyverno-test-duplicated-with-resources.yaml

@@ -1,19 +0,0 @@
-name: dummy-policy
-policies:
-  - dummy-policy.yaml
-resources:
-  - resource-duplicates.yaml
-
-results:
-  - policy: dummy-policy
-    rule: require-image-tag
-    resources:
-      - myapp-pod1
-    kind: Pod
-    result: pass
-  - policy: dummy-policy
-    rule: require-image-tag
-    resources:
-      - myapp-pod2
-    kind: Pod
-    result: pass

test/cli/test-unit/selectResourcesForCheck/kyverno-test-uniq-with-resource.yaml

@@ -1,19 +0,0 @@
-name: dummy-policy
-policies:
-  - dummy-policy.yaml
-resources:
-  - resource-uniq.yaml
-
-results:
-  - policy: dummy-policy
-    rule: require-image-tag
-    resources:
-      - myapp-pod1
-    kind: Pod
-    result: pass
-  - policy: dummy-policy
-    rule: require-image-tag
-    resources:
-      - myapp-pod2
-    kind: Pod
-    result: pass

test/cli/test-unit/selectResourcesForCheck/kyverno-test-uniq-with-resources.yaml

@@ -1,19 +0,0 @@
-name: dummy-policy
-policies:
-  - dummy-policy.yaml
-resources:
-  - resource-uniq.yaml
-
-results:
-  - policy: dummy-policy
-    rule: require-image-tag
-    resources:
-      - myapp-pod1
-    kind: Pod
-    result: pass
-  - policy: dummy-policy
-    rule: require-image-tag
-    resources:
-      - myapp-pod2
-    kind: Pod
-    result: pass