package test import ( "fmt" "os" policyreportv1alpha2 "github.com/kyverno/kyverno/api/policyreport/v1alpha2" "github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test/api" "github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test/manifest" sanitizederror "github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/sanitizedError" "github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/store" "github.com/kyverno/kyverno/pkg/openapi" "github.com/spf13/cobra" "sigs.k8s.io/controller-runtime/pkg/log" ) // Command returns version command func Command() *cobra.Command { var cmd *cobra.Command var testCase string var fileName, gitBranch string var registryAccess, failOnly, removeColor, manifestValidate, manifestMutate, compact bool cmd = &cobra.Command{ Use: "test [flags]\n kyverno test --git-branch \n kyverno test --manifest-mutate > kyverno-test.yaml\n kyverno test --manifest-validate > kyverno-test.yaml", // Args: cobra.ExactArgs(1), Short: "Run tests from directory.", Long: longHelp, Example: exampleHelp, RunE: func(cmd *cobra.Command, dirPath []string) (err error) { initColors(removeColor) defer func() { if err != nil { if !sanitizederror.IsErrorSanitized(err) { log.Log.Error(err, "failed to sanitize") err = fmt.Errorf("internal error") } } }() if manifestMutate { manifest.PrintMutate() } else if manifestValidate { manifest.PrintValidate() } else { store.SetRegistryAccess(registryAccess) _, err = testCommandExecute(dirPath, fileName, gitBranch, testCase, failOnly, false, compact) if err != nil { log.Log.V(3).Info("a directory is required") return err } } return nil }, } cmd.Flags().StringVarP(&fileName, "file-name", "f", "kyverno-test.yaml", "test filename") cmd.Flags().StringVarP(&gitBranch, "git-branch", "b", "", "test github repository branch") cmd.Flags().StringVarP(&testCase, "test-case-selector", "t", "", `run some specific test cases by passing a string argument in double quotes to this flag like - "policy=, rule=, resource= 0 && log.Log.V(1).Enabled() { fmt.Println("test errors:") for _, e := range errors { fmt.Printf(" %v \n", e.Error()) } } if !failOnly { fmt.Printf("\nTest Summary: %d tests passed and %d tests failed\n", rc.Pass+rc.Skip, rc.Fail) } else { fmt.Printf("\nTest Summary: %d out of %d tests failed\n", rc.Fail, rc.Pass+rc.Skip+rc.Fail) } fmt.Println() if rc.Fail > 0 && !failOnly { printFailedTestResult(table, compact) os.Exit(1) } os.Exit(0) return rc, nil } func printTestResult(resps map[string]policyreportv1alpha2.PolicyReportResult, testResults []api.TestResults, rc *resultCounts, failOnly bool, compact bool) (Table, error) { printer := newTablePrinter() var table Table var countDeprecatedResource int testCount := 1 for _, v := range testResults { var row Row row.ID = testCount if v.Resources == nil { testCount++ } row.Policy = boldFgCyan.Sprint(v.Policy) row.Rule = boldFgCyan.Sprint(v.Rule) if v.Resources != nil { for _, resource := range v.Resources { row.ID = testCount testCount++ row.Resource = boldFgCyan.Sprint(v.Namespace) + "/" + boldFgCyan.Sprint(v.Kind) + "/" + boldFgCyan.Sprint(resource) var ruleNameInResultKey string if v.AutoGeneratedRule != "" { ruleNameInResultKey = fmt.Sprintf("%s-%s", v.AutoGeneratedRule, v.Rule) } else { ruleNameInResultKey = v.Rule } resultKey := fmt.Sprintf("%s-%s-%s-%s", v.Policy, ruleNameInResultKey, v.Kind, resource) found, _ := isNamespacedPolicy(v.Policy) var ns string ns, v.Policy = getUserDefinedPolicyNameAndNamespace(v.Policy) if found && v.Namespace != "" { resultKey = fmt.Sprintf("%s-%s-%s-%s-%s-%s", ns, v.Policy, ruleNameInResultKey, v.Namespace, v.Kind, resource) } else if found { resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", ns, v.Policy, ruleNameInResultKey, v.Kind, resource) row.Policy = boldFgCyan.Sprint(ns) + "/" + boldFgCyan.Sprint(v.Policy) row.Resource = boldFgCyan.Sprint(v.Namespace) + "/" + boldFgCyan.Sprint(v.Kind) + "/" + boldFgCyan.Sprint(resource) } else if v.Namespace != "" { row.Resource = boldFgCyan.Sprint(v.Namespace) + "/" + boldFgCyan.Sprint(v.Kind) + "/" + boldFgCyan.Sprint(resource) resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", v.Policy, ruleNameInResultKey, v.Namespace, v.Kind, resource) } var testRes policyreportv1alpha2.PolicyReportResult if val, ok := resps[resultKey]; ok { testRes = val } else { log.Log.V(2).Info("result not found", "key", resultKey) row.Result = boldYellow.Sprint("Not found") rc.Fail++ row.isFailure = true table.Add(row) continue } row.Message = testRes.Message if v.Result == "" && v.Status != "" { v.Result = v.Status } if testRes.Result == v.Result { row.Result = boldGreen.Sprint("Pass") if testRes.Result == policyreportv1alpha2.StatusSkip { rc.Skip++ } else { rc.Pass++ } } else { log.Log.V(2).Info("result mismatch", "expected", v.Result, "received", testRes.Result, "key", resultKey) row.Result = boldRed.Sprint("Fail") rc.Fail++ row.isFailure = true } if failOnly { if row.Result == boldRed.Sprintf("Fail") || row.Result == "Fail" { table.Add(row) } } else { table.Add(row) } } } else if v.Resource != "" { countDeprecatedResource++ row.Resource = boldFgCyan.Sprint(v.Namespace) + "/" + boldFgCyan.Sprint(v.Kind) + "/" + boldFgCyan.Sprint(v.Resource) var ruleNameInResultKey string if v.AutoGeneratedRule != "" { ruleNameInResultKey = fmt.Sprintf("%s-%s", v.AutoGeneratedRule, v.Rule) } else { ruleNameInResultKey = v.Rule } resultKey := fmt.Sprintf("%s-%s-%s-%s", v.Policy, ruleNameInResultKey, v.Kind, v.Resource) found, _ := isNamespacedPolicy(v.Policy) var ns string ns, v.Policy = getUserDefinedPolicyNameAndNamespace(v.Policy) if found && v.Namespace != "" { resultKey = fmt.Sprintf("%s-%s-%s-%s-%s-%s", ns, v.Policy, ruleNameInResultKey, v.Namespace, v.Kind, v.Resource) } else if found { resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", ns, v.Policy, ruleNameInResultKey, v.Kind, v.Resource) row.Policy = boldFgCyan.Sprint(ns) + "/" + boldFgCyan.Sprint(v.Policy) row.Resource = boldFgCyan.Sprint(v.Namespace) + "/" + boldFgCyan.Sprint(v.Kind) + "/" + boldFgCyan.Sprint(v.Resource) } else if v.Namespace != "" { row.Resource = boldFgCyan.Sprint(v.Namespace) + "/" + boldFgCyan.Sprint(v.Kind) + "/" + boldFgCyan.Sprint(v.Resource) resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", v.Policy, ruleNameInResultKey, v.Namespace, v.Kind, v.Resource) } var testRes policyreportv1alpha2.PolicyReportResult if val, ok := resps[resultKey]; ok { testRes = val } else { log.Log.V(2).Info("result not found", "key", resultKey) row.Result = boldYellow.Sprint("Not found") rc.Fail++ row.isFailure = true table.Add(row) continue } row.Message = testRes.Message if v.Result == "" && v.Status != "" { v.Result = v.Status } if testRes.Result == v.Result { row.Result = boldGreen.Sprint("Pass") if testRes.Result == policyreportv1alpha2.StatusSkip { rc.Skip++ } else { rc.Pass++ } } else { log.Log.V(2).Info("result mismatch", "expected", v.Result, "received", testRes.Result, "key", resultKey) row.Result = boldRed.Sprint("Fail") rc.Fail++ row.isFailure = true } if failOnly { if row.Result == boldRed.Sprintf("Fail") || row.Result == "Fail" { table.Add(row) } } else { table.Add(row) } } } fmt.Printf("\n") printer.Print(table.Rows(compact)) return table, nil } func printFailedTestResult(table Table, compact bool) { printer := newTablePrinter() for i := range table.rows { table.rows[i].ID = i + 1 } fmt.Printf("Aggregated Failed Test Cases : ") fmt.Println() printer.Print(table.Rows(compact)) }