Add new test-case-selector flag to test command (#3183)

* added new test-case flag to test command
Signed-off-by: Shubham Nazare <shubham4443@gmail.com>

Co-authored-by: Vyankatesh Kudtarkar <vyankateshkd@gmail.com>
Co-authored-by: Sambhav Kothari <skothari44@bloomberg.net>

Makefile

@@ -225,7 +225,7 @@ test-clean:
 	go clean -testcache ./...
 
 .PHONY: test-cli
-test-cli: test-cli-policies test-cli-local test-cli-local-mutate
+test-cli: test-cli-policies test-cli-local test-cli-local-mutate test-cli-test-case-selector-flag
 
 .PHONY: test-cli-policies
 test-cli-policies: cli
@@ -239,6 +239,9 @@ test-cli-local: cli
 test-cli-local-mutate: cli
 	cmd/cli/kubectl-kyverno/kyverno test ./test/cli/test
 
+.PHONY: test-cli-test-case-selector-flag
+test-cli-test-case-selector-flag: cli
+	cmd/cli/kubectl-kyverno/kyverno test ./test/cli/test --test-case-selector "policy=disallow-latest-tag, rule=require-image-tag, resource=test-require-image-tag-pass"
 
 # go get downloads and installs the binary
 # we temporarily add the GO_ACC to the path

pkg/kyverno/test/test_command.go

@@ -16,6 +16,7 @@ import (
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/memfs"
 	"github.com/kataras/tablewriter"
+	v1 "github.com/kyverno/kyverno/api/kyverno/v1"
 	report "github.com/kyverno/kyverno/api/policyreport/v1alpha2"
 	client "github.com/kyverno/kyverno/pkg/dclient"
 	"github.com/kyverno/kyverno/pkg/engine/response"
@@ -77,6 +78,21 @@ applying 1 policy to 4 resources...
 
 Test Summary: 4 tests passed and 0 tests failed
 
+# Test some specific test cases out of many test cases in a local folder.
+kyverno test . --test-case-selector "policy=disallow-latest-tag, rule=require-image-tag, resource=test-require-image-tag-pass"
+
+Executing test-simple...
+applying 1 policy to 1 resource... 
+
+│───│─────────────────────│───────────────────│─────────────────────────────────────────│────────│
+│ # │ POLICY              │ RULE              │ RESOURCE                                │ RESULT │
+│───│─────────────────────│───────────────────│─────────────────────────────────────────│────────│
+│ 1 │ disallow-latest-tag │ require-image-tag │ default/Pod/test-require-image-tag-pass │ Pass   │
+│───│─────────────────────│───────────────────│─────────────────────────────────────────│────────│
+
+Test Summary: 1 tests passed and 0 tests failed
+
+
 
 **TEST FILE STRUCTURE**:
 
@@ -135,8 +151,9 @@ For more information visit https://kyverno.io/docs/kyverno-cli/#test
 // Command returns version command
 func Command() *cobra.Command {
 	var cmd *cobra.Command
+	var testCase string
 	var testFile []byte
-	var valuesFile, fileName, gitBranch string
+	var fileName, gitBranch string
 	cmd = &cobra.Command{
 		Use: "test <path_to_folder_Containing_test.yamls> [flags]\n  kyverno test <path_to_gitRepository_with_dir> --git-branch <branchName>\n  kyverno test --manifest-mutate > kyverno-test.yaml\n  kyverno test --manifest-validate > kyverno-test.yaml",
 		// Args:    cobra.ExactArgs(1),
@@ -194,7 +211,7 @@ results:
 				fmt.Println(string(testFile))
 				return nil
 			}
-			_, err = testCommandExecute(dirPath, valuesFile, fileName, gitBranch)
+			_, err = testCommandExecute(dirPath, fileName, gitBranch, testCase)
 			if err != nil {
 				log.Log.V(3).Info("a directory is required")
 				return err
@@ -205,7 +222,7 @@ results:
 	}
 	cmd.Flags().StringVarP(&fileName, "file-name", "f", "kyverno-test.yaml", "test filename")
 	cmd.Flags().StringVarP(&gitBranch, "git-branch", "b", "", "test github repository branch")
-
+	cmd.Flags().StringVarP(&testCase, "test-case-selector", "t", "", `run some specific test cases by passing a string argument in double quotes to this flag like - "policy=<policy_name>, rule=<rule_name>, resource=<resource_name". The argument could be any combination of policy, rule and resource.`)
 	cmd.Flags().BoolP("manifest-mutate", "", false, "prints out a template test manifest for a mutate policy")
 	cmd.Flags().BoolP("manifest-validate", "", false, "prints out a template test manifest for a validate policy")
 	return cmd
@@ -263,16 +280,57 @@ type resultCounts struct {
 	Fail int
 }
 
-func testCommandExecute(dirPath []string, valuesFile string, fileName string, gitBranch string) (rc *resultCounts, err error) {
+type testFilter struct {
+	policy   string
+	rule     string
+	resource string
+	enabled  bool
+}
+
+func testCommandExecute(dirPath []string, fileName string, gitBranch string, testCase string) (rc *resultCounts, err error) {
 	var errors []error
 	fs := memfs.New()
 	rc = &resultCounts{}
 	var testYamlCount int
 	var testYamlNameCount int
+	var tf = &testFilter{
+		enabled: true,
+	}
 
 	if len(dirPath) == 0 {
 		return rc, sanitizederror.NewWithError(fmt.Sprintf("a directory is required"), err)
 	}
+
+	if len(testCase) != 0 {
+		parameters := map[string]string{"policy": "", "rule": "", "resource": ""}
+
+		for _, t := range strings.Split(testCase, ",") {
+			if !strings.Contains(t, "=") {
+				fmt.Printf("\n Invalid test-case-selector argument. Selecting all test cases. \n")
+				tf.enabled = false
+				break
+			}
+
+			key := strings.TrimSpace(strings.Split(t, "=")[0])
+			value := strings.TrimSpace(strings.Split(t, "=")[1])
+
+			_, ok := parameters[key]
+			if !ok {
+				fmt.Printf("\n Invalid parameter. Parameter can only be policy, rule or resource. Selecting all test cases \n")
+				tf.enabled = false
+				break
+			}
+
+			parameters[key] = value
+		}
+
+		tf.policy = parameters["policy"]
+		tf.rule = parameters["rule"]
+		tf.resource = parameters["resource"]
+	} else {
+		tf.enabled = false
+	}
+
 	openAPIController, err := openapi.NewOpenAPIController()
 	if err != nil {
 		return rc, fmt.Errorf("unable to create open api controller, %w", err)
@@ -354,7 +412,7 @@ func testCommandExecute(dirPath []string, valuesFile string, fileName string, gi
 					continue
 				}
 
-				if err := applyPoliciesFromPath(fs, policyBytes, valuesFile, true, policyresoucePath, rc, openAPIController); err != nil {
+				if err := applyPoliciesFromPath(fs, policyBytes, true, policyresoucePath, rc, openAPIController, tf); err != nil {
 					return rc, sanitizederror.NewWithError("failed to apply test command", err)
 				}
 			}
@@ -371,7 +429,7 @@ func testCommandExecute(dirPath []string, valuesFile string, fileName string, gi
 		var testFiles int
 		var deprecatedFiles int
 		path := filepath.Clean(dirPath[0])
-		errors = getLocalDirTestFiles(fs, path, fileName, valuesFile, rc, &testFiles, &deprecatedFiles, openAPIController)
+		errors = getLocalDirTestFiles(fs, path, fileName, rc, &testFiles, &deprecatedFiles, openAPIController, tf)
 
 		if testFiles == 0 {
 			fmt.Printf("\n No test files found. Please provide test YAML files named kyverno-test.yaml \n")
@@ -398,7 +456,7 @@ func testCommandExecute(dirPath []string, valuesFile string, fileName string, gi
 	return rc, nil
 }
 
-func getLocalDirTestFiles(fs billy.Filesystem, path, fileName, valuesFile string, rc *resultCounts, testFiles *int, deprecatedFiles *int, openAPIController *openapi.Controller) []error {
+func getLocalDirTestFiles(fs billy.Filesystem, path, fileName string, rc *resultCounts, testFiles *int, deprecatedFiles *int, openAPIController *openapi.Controller, tf *testFilter) []error {
 	var errors []error
 
 	files, err := ioutil.ReadDir(path)
@@ -407,7 +465,7 @@ func getLocalDirTestFiles(fs billy.Filesystem, path, fileName, valuesFile string
 	}
 	for _, file := range files {
 		if file.IsDir() {
-			getLocalDirTestFiles(fs, filepath.Join(path, file.Name()), fileName, valuesFile, rc, testFiles, deprecatedFiles, openAPIController)
+			getLocalDirTestFiles(fs, filepath.Join(path, file.Name()), fileName, rc, testFiles, deprecatedFiles, openAPIController, tf)
 			continue
 		}
 		if strings.Contains(file.Name(), fileName) || strings.Contains(file.Name(), "test.yaml") {
@@ -426,7 +484,7 @@ func getLocalDirTestFiles(fs billy.Filesystem, path, fileName, valuesFile string
 				errors = append(errors, sanitizederror.NewWithError("failed to convert json", err))
 				continue
 			}
-			if err := applyPoliciesFromPath(fs, valuesBytes, valuesFile, false, path, rc, openAPIController); err != nil {
+			if err := applyPoliciesFromPath(fs, valuesBytes, false, path, rc, openAPIController, tf); err != nil {
 				errors = append(errors, sanitizederror.NewWithError(fmt.Sprintf("failed to apply test command from file %s", file.Name()), err))
 				continue
 			}
@@ -662,7 +720,7 @@ func getFullPath(paths []string, policyResourcePath string, isGit bool) []string
 	return paths
 }
 
-func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, valuesFile string, isGit bool, policyResourcePath string, rc *resultCounts, openAPIController *openapi.Controller) (err error) {
+func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, isGit bool, policyResourcePath string, rc *resultCounts, openAPIController *openapi.Controller, tf *testFilter) (err error) {
 
 	engineResponses := make([]*response.EngineResponse, 0)
 	var dClient *client.Client
@@ -676,8 +734,22 @@ func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, valuesFile s
 		return sanitizederror.NewWithError("failed to decode yaml", err)
 	}
 
+	if tf.enabled {
+		var filteredResults []TestResults
+		for _, res := range values.Results {
+			if (len(tf.policy) == 0 || tf.policy == res.Policy) && (len(tf.resource) == 0 || tf.resource == res.Resource) && (len(tf.rule) == 0 || tf.rule == res.Rule) {
+				filteredResults = append(filteredResults, res)
+			}
+		}
+		values.Results = filteredResults
+	}
+
+	if len(values.Results) == 0 {
+		return nil
+	}
+
 	fmt.Printf("\nExecuting %s...", values.Name)
-	valuesFile = values.Variables
+	valuesFile := values.Variables
 	variables, globalValMap, valuesMap, namespaceSelectorMap, err := common.GetVariable(variablesString, values.Variables, fs, isGit, policyResourcePath)
 	if err != nil {
 		if !sanitizederror.IsErrorSanitized(err) {
@@ -701,6 +773,31 @@ func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, valuesFile s
 		os.Exit(1)
 	}
 
+	var filteredPolicies = []*v1.ClusterPolicy{}
+	for _, p := range policies {
+		for _, res := range values.Results {
+			if p.Name == res.Policy {
+				filteredPolicies = append(filteredPolicies, p)
+				break
+			}
+		}
+	}
+
+	for _, p := range filteredPolicies {
+		var filteredRules = []v1.Rule{}
+
+		for _, rule := range p.Spec.Rules {
+			for _, res := range values.Results {
+				if rule.Name == res.Rule {
+					filteredRules = append(filteredRules, rule)
+					break
+				}
+			}
+		}
+		p.Spec.Rules = filteredRules
+	}
+	policies = filteredPolicies
+
 	mutatedPolicies, err := common.MutatePolicies(policies)
 	if err != nil {
 		if !sanitizederror.IsErrorSanitized(err) {
@@ -719,6 +816,17 @@ func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, valuesFile s
 		os.Exit(1)
 	}
 
+	var filteredResources = []*unstructured.Unstructured{}
+	for _, r := range resources {
+		for _, res := range values.Results {
+			if r.GetName() == res.Resource {
+				filteredResources = append(filteredResources, r)
+				break
+			}
+		}
+	}
+	resources = filteredResources
+
 	msgPolicies := "1 policy"
 	if len(mutatedPolicies) > 1 {
 		msgPolicies = fmt.Sprintf("%d policies", len(policies))