From be6cc1381fc6343fb4247d02e4b22fd07de3dc58 Mon Sep 17 00:00:00 2001 From: Anutosh Bhat <87052487+anutosh491@users.noreply.github.com> Date: Fri, 22 Jul 2022 20:02:12 +0530 Subject: [PATCH] Implementing flag to show all failing tests only through the test command (#4227) * fix kyverno cli policy-report typo (#4224) - fix kyverno cli policy report typo - add shorthand for policy-report flag Signed-off-by: Vyankatesh vyankateshkd@gmail.com Signed-off-by: anutosh491 * Added flag for getting fail only tests Signed-off-by: anutosh491 * Improve output message Signed-off-by: anutosh491 * Updated flag name as per CamelCase Signed-off-by: anutosh491 Co-authored-by: Vyankatesh Kudtarkar --- cmd/cli/kubectl-kyverno/test/test_command.go | 47 +++++++++++++------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/cmd/cli/kubectl-kyverno/test/test_command.go b/cmd/cli/kubectl-kyverno/test/test_command.go index af14a53df6..f712c58a4b 100644 --- a/cmd/cli/kubectl-kyverno/test/test_command.go +++ b/cmd/cli/kubectl-kyverno/test/test_command.go @@ -157,7 +157,7 @@ func Command() *cobra.Command { var testCase string var testFile []byte var fileName, gitBranch string - var registryAccess bool + var registryAccess, failOnly bool cmd = &cobra.Command{ Use: "test [flags]\n kyverno test --git-branch \n kyverno test --manifest-mutate > kyverno-test.yaml\n kyverno test --manifest-validate > kyverno-test.yaml", // Args: cobra.ExactArgs(1), @@ -216,7 +216,7 @@ results: return nil } store.SetRegistryAccess(registryAccess) - _, err = testCommandExecute(dirPath, fileName, gitBranch, testCase) + _, err = testCommandExecute(dirPath, fileName, gitBranch, testCase, failOnly) if err != nil { log.Log.V(3).Info("a directory is required") return err @@ -231,6 +231,7 @@ results: cmd.Flags().BoolP("manifest-mutate", "", false, "prints out a template test manifest for a mutate policy") cmd.Flags().BoolP("manifest-validate", "", false, "prints out a template test manifest for a validate policy") cmd.Flags().BoolVarP(®istryAccess, "registry", "", false, "If set to true, access the image registry using local docker credentials to populate external data") + cmd.Flags().BoolVarP(&failOnly, "fail-only", "", false, "If set to true, display all the failing test only as output for the test command") return cmd } @@ -317,7 +318,7 @@ type testFilter struct { var ftable = []Table{} -func testCommandExecute(dirPath []string, fileName string, gitBranch string, testCase string) (rc *resultCounts, err error) { +func testCommandExecute(dirPath []string, fileName string, gitBranch string, testCase string, failOnly bool) (rc *resultCounts, err error) { var errors []error fs := memfs.New() rc = &resultCounts{} @@ -437,7 +438,7 @@ func testCommandExecute(dirPath []string, fileName string, gitBranch string, tes errors = append(errors, sanitizederror.NewWithError("failed to convert to JSON", err)) continue } - if err := applyPoliciesFromPath(fs, policyBytes, true, policyresoucePath, rc, openAPIController, tf); err != nil { + if err := applyPoliciesFromPath(fs, policyBytes, true, policyresoucePath, rc, openAPIController, tf, failOnly); err != nil { return rc, sanitizederror.NewWithError("failed to apply test command", err) } } @@ -449,7 +450,7 @@ func testCommandExecute(dirPath []string, fileName string, gitBranch string, tes } else { var testFiles int path := filepath.Clean(dirPath[0]) - errors = getLocalDirTestFiles(fs, path, fileName, rc, &testFiles, openAPIController, tf) + errors = getLocalDirTestFiles(fs, path, fileName, rc, &testFiles, openAPIController, tf, failOnly) if testFiles == 0 { fmt.Printf("\n No test files found. Please provide test YAML files named kyverno-test.yaml \n") @@ -463,10 +464,14 @@ func testCommandExecute(dirPath []string, fileName string, gitBranch string, tes } } - fmt.Printf("\nTest Summary: %d tests passed and %d tests failed\n", rc.Pass+rc.Skip, rc.Fail) + if !failOnly { + fmt.Printf("\nTest Summary: %d tests passed and %d tests failed\n", rc.Pass+rc.Skip, rc.Fail) + } else { + fmt.Printf("\nTest Summary: %d out of %d tests failed\n", rc.Fail, rc.Pass+rc.Skip+rc.Fail) + } fmt.Printf("\n") - if rc.Fail > 0 { + if rc.Fail > 0 && !failOnly { printFailedTestResult() os.Exit(1) } @@ -474,7 +479,7 @@ func testCommandExecute(dirPath []string, fileName string, gitBranch string, tes return rc, nil } -func getLocalDirTestFiles(fs billy.Filesystem, path, fileName string, rc *resultCounts, testFiles *int, openAPIController *openapi.Controller, tf *testFilter) []error { +func getLocalDirTestFiles(fs billy.Filesystem, path, fileName string, rc *resultCounts, testFiles *int, openAPIController *openapi.Controller, tf *testFilter, failOnly bool) []error { var errors []error files, err := ioutil.ReadDir(path) @@ -483,7 +488,7 @@ func getLocalDirTestFiles(fs billy.Filesystem, path, fileName string, rc *result } for _, file := range files { if file.IsDir() { - getLocalDirTestFiles(fs, filepath.Join(path, file.Name()), fileName, rc, testFiles, openAPIController, tf) + getLocalDirTestFiles(fs, filepath.Join(path, file.Name()), fileName, rc, testFiles, openAPIController, tf, failOnly) continue } if file.Name() == fileName { @@ -499,7 +504,7 @@ func getLocalDirTestFiles(fs billy.Filesystem, path, fileName string, rc *result errors = append(errors, sanitizederror.NewWithError("failed to convert json", err)) continue } - if err := applyPoliciesFromPath(fs, valuesBytes, false, path, rc, openAPIController, tf); err != nil { + if err := applyPoliciesFromPath(fs, valuesBytes, false, path, rc, openAPIController, tf, failOnly); err != nil { errors = append(errors, sanitizederror.NewWithError(fmt.Sprintf("failed to apply test command from file %s", file.Name()), err)) continue } @@ -813,7 +818,7 @@ func getFullPath(paths []string, policyResourcePath string, isGit bool) []string return paths } -func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, isGit bool, policyResourcePath string, rc *resultCounts, openAPIController *openapi.Controller, tf *testFilter) (err error) { +func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, isGit bool, policyResourcePath string, rc *resultCounts, openAPIController *openapi.Controller, tf *testFilter, failOnly bool) (err error) { engineResponses := make([]*response.EngineResponse, 0) var dClient dclient.Interface values := &Test{} @@ -1014,7 +1019,7 @@ func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, isGit bool, } } resultsMap, testResults := buildPolicyResults(engineResponses, values.Results, pvInfos, policyResourcePath, fs, isGit) - resultErr := printTestResult(resultsMap, testResults, rc) + resultErr := printTestResult(resultsMap, testResults, rc, failOnly) if resultErr != nil { return sanitizederror.NewWithError("failed to print test result:", resultErr) } @@ -1022,7 +1027,7 @@ func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, isGit bool, return } -func printTestResult(resps map[string]policyreportv1alpha2.PolicyReportResult, testResults []TestResults, rc *resultCounts) error { +func printTestResult(resps map[string]policyreportv1alpha2.PolicyReportResult, testResults []TestResults, rc *resultCounts, failOnly bool) error { printer := tableprinter.New(os.Stdout) table := []Table{} boldGreen := color.New(color.FgGreen).Add(color.Bold) @@ -1094,7 +1099,13 @@ func printTestResult(resps map[string]policyreportv1alpha2.PolicyReportResult, t ftable = append(ftable, *res) } - table = append(table, *res) + if failOnly { + if res.Result == boldRed.Sprintf("Fail") { + table = append(table, *res) + } + } else { + table = append(table, *res) + } } } else if v.Resource != "" { countDeprecatedResource++ @@ -1153,7 +1164,13 @@ func printTestResult(resps map[string]policyreportv1alpha2.PolicyReportResult, t ftable = append(ftable, *res) } - table = append(table, *res) + if failOnly { + if res.Result == boldRed.Sprintf("Fail") { + table = append(table, *res) + } + } else { + table = append(table, *res) + } } }