refactor: restructure cli test command (#6942)

* refactor: restructure cli test command

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

cmd/cli/kubectl-kyverno/main.go

@@ -10,10 +10,8 @@ import (
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/oci"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/version"
+	"github.com/kyverno/kyverno/pkg/logging"
 	"github.com/spf13/cobra"
-	"k8s.io/klog/v2"
-	"k8s.io/klog/v2/klogr"
-	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
 const EnableExperimentalEnv = "KYVERNO_EXPERIMENTAL"
@@ -21,30 +19,27 @@ const EnableExperimentalEnv = "KYVERNO_EXPERIMENTAL"
 func main() {
 	cli := &cobra.Command{
 		Use:   "kyverno",
-		Long:  `To enable experimental commands, KYVERNO_EXPERIMENTAL should be configured with true or 1.`,
+		Long:  "To enable experimental commands, KYVERNO_EXPERIMENTAL should be configured with true or 1.",
 		Short: "Kubernetes Native Policy Management",
 	}
-
+	configureLogs(cli)
-	configurelog(cli)
+	registerCommands(cli)
-
-	commands := []*cobra.Command{
-		version.Command(),
-		apply.Command(),
-		test.Command(),
-		jp.Command(),
-	}
-
-	if enableExperimental() {
-		commands = append(commands, oci.Command())
-	}
-
-	cli.AddCommand(commands...)
-
 	if err := cli.Execute(); err != nil {
 		os.Exit(1)
 	}
 }
 
+func configureLogs(cli *cobra.Command) {
+	logging.InitFlags(nil)
+	cli.PersistentFlags().AddGoFlagSet(flag.CommandLine)
+	_ = cli.PersistentFlags().MarkHidden("alsologtostderr")
+	_ = cli.PersistentFlags().MarkHidden("logtostderr")
+	_ = cli.PersistentFlags().MarkHidden("log_dir")
+	_ = cli.PersistentFlags().MarkHidden("log_backtrace_at")
+	_ = cli.PersistentFlags().MarkHidden("stderrthreshold")
+	_ = cli.PersistentFlags().MarkHidden("vmodule")
+}
+
 func enableExperimental() bool {
 	if b, err := strconv.ParseBool(os.Getenv(EnableExperimentalEnv)); err == nil {
 		return b
@@ -52,20 +47,9 @@ func enableExperimental() bool {
 	return false
 }
 
-func configurelog(cli *cobra.Command) {
+func registerCommands(cli *cobra.Command) {
-	// clear flags initialized in static dependencies
+	cli.AddCommand(version.Command(), apply.Command(), test.Command(), jp.Command())
-	if flag.CommandLine.Lookup("log_dir") != nil {
+	if enableExperimental() {
-		flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+		cli.AddCommand(oci.Command())
 	}
-
-	klog.InitFlags(nil)
-	cli.PersistentFlags().AddGoFlagSet(flag.CommandLine)
-	log.SetLogger(klogr.New())
-
-	_ = cli.PersistentFlags().MarkHidden("alsologtostderr")
-	_ = cli.PersistentFlags().MarkHidden("logtostderr")
-	_ = cli.PersistentFlags().MarkHidden("log_dir")
-	_ = cli.PersistentFlags().MarkHidden("log_backtrace_at")
-	_ = cli.PersistentFlags().MarkHidden("stderrthreshold")
-	_ = cli.PersistentFlags().MarkHidden("vmodule")
 }

cmd/cli/kubectl-kyverno/test/filter.go

@@ -15,30 +15,32 @@ func noFilter(api.TestResults) bool {
 
 func parseFilter(in string) filter {
 	var filters []filter
-	for _, t := range strings.Split(in, ",") {
+	if in != "" {
-		parts := strings.Split(t, "=")
+		for _, t := range strings.Split(in, ",") {
-		if len(parts) != 2 {
+			parts := strings.Split(t, "=")
-			fmt.Printf("\n Invalid test-case-selector argument. Selecting all test cases. \n")
+			if len(parts) != 2 {
-			return noFilter
+				fmt.Printf("\n Invalid test-case-selector argument (%s). Selecting all test cases. \n", t)
-		}
+				return noFilter
-		key := strings.TrimSpace(parts[0])
+			}
-		value := strings.TrimSpace(parts[1])
+			key := strings.TrimSpace(parts[0])
-		switch key {
+			value := strings.TrimSpace(parts[1])
-		case "policy":
+			switch key {
-			filters = append(filters, func(r api.TestResults) bool {
+			case "policy":
-				return r.Policy == "" || r.Policy == value
+				filters = append(filters, func(r api.TestResults) bool {
-			})
+					return r.Policy == "" || r.Policy == value
-		case "rule":
+				})
-			filters = append(filters, func(r api.TestResults) bool {
+			case "rule":
-				return r.Rule == "" || r.Rule == value
+				filters = append(filters, func(r api.TestResults) bool {
-			})
+					return r.Rule == "" || r.Rule == value
-		case "resource":
+				})
-			filters = append(filters, func(r api.TestResults) bool {
+			case "resource":
-				return r.Resource == "" || r.Resource == value
+				filters = append(filters, func(r api.TestResults) bool {
-			})
+					return r.Resource == "" || r.Resource == value
-		default:
+				})
-			fmt.Printf("\n Invalid parameter. Parameter can only be policy, rule or resource. Selecting all test cases \n")
+			default:
-			return noFilter
+				fmt.Printf("\n Invalid parameter. Parameter can only be policy, rule or resource. Selecting all test cases \n")
+				return noFilter
+			}
 		}
 	}
 	return func(r api.TestResults) bool {

cmd/cli/kubectl-kyverno/test/load.go

@@ -0,0 +1,145 @@
+package test
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/go-git/go-billy/v5"
+	"github.com/go-git/go-billy/v5/memfs"
+	sanitizederror "github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/sanitizedError"
+	gitutils "github.com/kyverno/kyverno/pkg/utils/git"
+	"k8s.io/apimachinery/pkg/util/yaml"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+type policy struct {
+	bytes        []byte
+	resourcePath string
+}
+
+func loadTests(
+	dirPath []string,
+	fileName string,
+	gitBranch string,
+) (billy.Filesystem, []policy, []error) {
+	var policies []policy
+	var errors []error
+	if strings.Contains(dirPath[0], "https://") {
+		fs := memfs.New()
+		if gitURL, err := url.Parse(dirPath[0]); err != nil {
+			errors = append(errors, sanitizederror.NewWithError("failed to parse URL", err))
+		} else {
+			pathElems := strings.Split(gitURL.Path[1:], "/")
+			if len(pathElems) <= 1 {
+				err := fmt.Errorf("invalid URL path %s - expected https://github.com/:owner/:repository/:branch (without --git-branch flag) OR https://github.com/:owner/:repository/:directory (with --git-branch flag)", gitURL.Path)
+				fmt.Printf("Error: failed to parse URL \nCause: %s\n", err)
+				os.Exit(1)
+			}
+			gitURL.Path = strings.Join([]string{pathElems[0], pathElems[1]}, "/")
+			repoURL := gitURL.String()
+			var gitPathToYamls string
+			if gitBranch == "" {
+				gitPathToYamls = "/"
+				if string(dirPath[0][len(dirPath[0])-1]) == "/" {
+					gitBranch = strings.ReplaceAll(dirPath[0], repoURL+"/", "")
+				} else {
+					gitBranch = strings.ReplaceAll(dirPath[0], repoURL, "")
+				}
+				if gitBranch == "" {
+					gitBranch = "main"
+				} else if string(gitBranch[0]) == "/" {
+					gitBranch = gitBranch[1:]
+				}
+			} else {
+				if string(dirPath[0][len(dirPath[0])-1]) == "/" {
+					gitPathToYamls = strings.ReplaceAll(dirPath[0], repoURL+"/", "/")
+				} else {
+					gitPathToYamls = strings.ReplaceAll(dirPath[0], repoURL, "/")
+				}
+			}
+			_, cloneErr := gitutils.Clone(repoURL, fs, gitBranch)
+			if cloneErr != nil {
+				fmt.Printf("Error: failed to clone repository \nCause: %s\n", cloneErr)
+				log.Log.V(3).Info(fmt.Sprintf("failed to clone repository  %v as it is not valid", repoURL), "error", cloneErr)
+				os.Exit(1)
+			}
+			if policyYamls, err := gitutils.ListYamls(fs, gitPathToYamls); err != nil {
+				errors = append(errors, sanitizederror.NewWithError("failed to list YAMLs in repository", err))
+			} else {
+				sort.Strings(policyYamls)
+				for _, yamlFilePath := range policyYamls {
+					file, err := fs.Open(yamlFilePath)
+					if err != nil {
+						errors = append(errors, sanitizederror.NewWithError("Error: failed to open file", err))
+						continue
+					}
+					if path.Base(file.Name()) == fileName {
+						policyresoucePath := strings.Trim(yamlFilePath, fileName)
+						bytes, err := io.ReadAll(file)
+						if err != nil {
+							errors = append(errors, sanitizederror.NewWithError("Error: failed to read file", err))
+							continue
+						}
+						policyBytes, err := yaml.ToJSON(bytes)
+						if err != nil {
+							errors = append(errors, sanitizederror.NewWithError("failed to convert to JSON", err))
+							continue
+						}
+						policies = append(policies, policy{
+							bytes:        policyBytes,
+							resourcePath: policyresoucePath,
+						})
+					}
+				}
+			}
+		}
+		return fs, policies, errors
+	} else {
+		path := filepath.Clean(dirPath[0])
+		policies, errors = loadLocalTest(path, fileName)
+		return nil, policies, errors
+	}
+}
+
+func loadLocalTest(
+	path string,
+	fileName string,
+) ([]policy, []error) {
+	var policies []policy
+	var errors []error
+	files, err := os.ReadDir(path)
+	if err != nil {
+		errors = append(errors, fmt.Errorf("failed to read %v: %v", path, err.Error()))
+	} else {
+		for _, file := range files {
+			if file.IsDir() {
+				ps, errs := loadLocalTest(filepath.Join(path, file.Name()), fileName)
+				policies = append(policies, ps...)
+				errors = append(errors, errs...)
+			} else if file.Name() == fileName {
+				// We accept the risk of including files here as we read the test dir only.
+				yamlFile, err := os.ReadFile(filepath.Join(path, file.Name())) // #nosec G304
+				if err != nil {
+					errors = append(errors, sanitizederror.NewWithError("unable to read yaml", err))
+					continue
+				}
+				valuesBytes, err := yaml.ToJSON(yamlFile)
+				if err != nil {
+					errors = append(errors, sanitizederror.NewWithError("failed to convert json", err))
+					continue
+				}
+				policies = append(policies, policy{
+					bytes:        valuesBytes,
+					resourcePath: path,
+				})
+			}
+		}
+	}
+	return policies, errors
+}

cmd/cli/kubectl-kyverno/test/output.go

@@ -9,32 +9,42 @@ import (
 )
 
 var (
-	boldGreen  = color.New(color.FgGreen).Add(color.Bold)
+	boldGreen     *color.Color
-	boldRed    = color.New(color.FgRed).Add(color.Bold)
+	boldRed       *color.Color
-	boldYellow = color.New(color.FgYellow).Add(color.Bold)
+	boldYellow    *color.Color
-	boldFgCyan = color.New(color.FgCyan).Add(color.Bold)
+	boldFgCyan    *color.Color
+	headerBgColor int
+	headerFgColor int
 )
 
-func colorize(noColor bool, color *color.Color, format string, a ...interface{}) string {
+func initColors(noColor bool) {
-	if noColor {
+	toggleColor := func(c *color.Color) *color.Color {
-		return format
+		if noColor {
+			c.DisableColor()
+		}
+		return c
+	}
+	boldGreen = toggleColor(color.New(color.FgGreen).Add(color.Bold))
+	boldRed = toggleColor(color.New(color.FgRed).Add(color.Bold))
+	boldYellow = toggleColor(color.New(color.FgYellow).Add(color.Bold))
+	boldFgCyan = toggleColor(color.New(color.FgCyan).Add(color.Bold))
+	if !noColor {
+		headerBgColor = tablewriter.BgBlackColor
+		headerFgColor = tablewriter.FgGreenColor
 	}
-	return color.Sprintf(format, a...)
 }
 
-func newTablePrinter(noColor bool) *tableprinter.Printer {
+func newTablePrinter() *tableprinter.Printer {
 	printer := tableprinter.New(os.Stdout)
 	printer.BorderTop, printer.BorderBottom, printer.BorderLeft, printer.BorderRight = true, true, true, true
 	printer.CenterSeparator = "│"
 	printer.ColumnSeparator = "│"
 	printer.RowSeparator = "─"
 	printer.RowCharLimit = 300
+	printer.HeaderBgColor = headerBgColor
+	printer.HeaderFgColor = headerFgColor
 	printer.RowLengthTitle = func(rowsLength int) bool {
 		return rowsLength > 10
 	}
-	if !noColor {
-		printer.HeaderBgColor = tablewriter.BgBlackColor
-		printer.HeaderFgColor = tablewriter.FgGreenColor
-	}
 	return printer
 }

cmd/cli/kubectl-kyverno/test/test.go

@@ -0,0 +1,600 @@
+package test
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/go-git/go-billy/v5"
+	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
+	"github.com/kyverno/kyverno/api/kyverno/v1beta1"
+	policyreportv1alpha2 "github.com/kyverno/kyverno/api/policyreport/v1alpha2"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test/api"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/common"
+	sanitizederror "github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/sanitizedError"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/store"
+	"github.com/kyverno/kyverno/pkg/autogen"
+	"github.com/kyverno/kyverno/pkg/background/generate"
+	"github.com/kyverno/kyverno/pkg/clients/dclient"
+	engineapi "github.com/kyverno/kyverno/pkg/engine/api"
+	"github.com/kyverno/kyverno/pkg/openapi"
+	policy2 "github.com/kyverno/kyverno/pkg/policy"
+	"golang.org/x/exp/slices"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+func applyPoliciesFromPath(
+	fs billy.Filesystem,
+	policyBytes []byte,
+	isGit bool,
+	policyResourcePath string,
+	rc *resultCounts,
+	openApiManager openapi.Manager,
+	filter filter,
+	auditWarn bool,
+) (map[string]policyreportv1alpha2.PolicyReportResult, []api.TestResults, error) {
+	engineResponses := make([]engineapi.EngineResponse, 0)
+	var dClient dclient.Interface
+	values := &api.Test{}
+	var variablesString string
+	var resultCounts common.ResultCounts
+
+	store.SetMock(true)
+	if err := json.Unmarshal(policyBytes, values); err != nil {
+		return nil, nil, sanitizederror.NewWithError("failed to decode yaml", err)
+	}
+
+	var filteredResults []api.TestResults
+	for _, res := range values.Results {
+		if filter(res) {
+			filteredResults = append(filteredResults, res)
+		}
+	}
+	values.Results = filteredResults
+
+	if len(values.Results) == 0 {
+		return nil, nil, nil
+	}
+
+	fmt.Printf("\nExecuting %s...\n", values.Name)
+	valuesFile := values.Variables
+	userInfoFile := values.UserInfo
+
+	variables, globalValMap, valuesMap, namespaceSelectorMap, subresources, err := common.GetVariable(variablesString, values.Variables, fs, isGit, policyResourcePath)
+	if err != nil {
+		if !sanitizederror.IsErrorSanitized(err) {
+			return nil, nil, sanitizederror.NewWithError("failed to decode yaml", err)
+		}
+		return nil, nil, err
+	}
+
+	// get the user info as request info from a different file
+	var userInfo v1beta1.RequestInfo
+
+	if userInfoFile != "" {
+		userInfo, err = common.GetUserInfoFromPath(fs, userInfoFile, isGit, policyResourcePath)
+		if err != nil {
+			fmt.Printf("Error: failed to load request info\nCause: %s\n", err)
+			os.Exit(1)
+		}
+	}
+
+	policyFullPath := getFullPath(values.Policies, policyResourcePath, isGit)
+	resourceFullPath := getFullPath(values.Resources, policyResourcePath, isGit)
+
+	for i, result := range values.Results {
+		arrPatchedResource := []string{result.PatchedResource}
+		arrGeneratedResource := []string{result.GeneratedResource}
+		arrCloneSourceResource := []string{result.CloneSourceResource}
+
+		patchedResourceFullPath := getFullPath(arrPatchedResource, policyResourcePath, isGit)
+		generatedResourceFullPath := getFullPath(arrGeneratedResource, policyResourcePath, isGit)
+		CloneSourceResourceFullPath := getFullPath(arrCloneSourceResource, policyResourcePath, isGit)
+
+		values.Results[i].PatchedResource = patchedResourceFullPath[0]
+		values.Results[i].GeneratedResource = generatedResourceFullPath[0]
+		values.Results[i].CloneSourceResource = CloneSourceResourceFullPath[0]
+	}
+
+	policies, err := common.GetPoliciesFromPaths(fs, policyFullPath, isGit, policyResourcePath)
+	if err != nil {
+		fmt.Printf("Error: failed to load policies\nCause: %s\n", err)
+		os.Exit(1)
+	}
+
+	var filteredPolicies []kyvernov1.PolicyInterface
+	for _, p := range policies {
+		for _, res := range values.Results {
+			if p.GetName() == res.Policy {
+				filteredPolicies = append(filteredPolicies, p)
+				break
+			}
+		}
+	}
+
+	ruleToCloneSourceResource := map[string]string{}
+	for _, p := range filteredPolicies {
+		var filteredRules []kyvernov1.Rule
+
+		for _, rule := range autogen.ComputeRules(p) {
+			for _, res := range values.Results {
+				if rule.Name == res.Rule {
+					filteredRules = append(filteredRules, rule)
+					if rule.HasGenerate() {
+						ruleUnstr, err := generate.GetUnstrRule(rule.Generation.DeepCopy())
+						if err != nil {
+							fmt.Printf("Error: failed to get unstructured rule\nCause: %s\n", err)
+							break
+						}
+
+						genClone, _, err := unstructured.NestedMap(ruleUnstr.Object, "clone")
+						if err != nil {
+							fmt.Printf("Error: failed to read data\nCause: %s\n", err)
+							break
+						}
+
+						if len(genClone) != 0 {
+							ruleToCloneSourceResource[rule.Name] = res.CloneSourceResource
+						}
+					}
+					break
+				}
+			}
+		}
+		p.GetSpec().SetRules(filteredRules)
+	}
+	policies = filteredPolicies
+
+	err = common.PrintMutatedPolicy(policies)
+	if err != nil {
+		return nil, nil, sanitizederror.NewWithError("failed to print mutated policy", err)
+	}
+
+	resources, err := common.GetResourceAccordingToResourcePath(fs, resourceFullPath, false, policies, dClient, "", false, isGit, policyResourcePath)
+	if err != nil {
+		fmt.Printf("Error: failed to load resources\nCause: %s\n", err)
+		os.Exit(1)
+	}
+
+	checkableResources := selectResourcesForCheck(resources, values)
+
+	msgPolicies := "1 policy"
+	if len(policies) > 1 {
+		msgPolicies = fmt.Sprintf("%d policies", len(policies))
+	}
+
+	msgResources := "1 resource"
+	if len(checkableResources) > 1 {
+		msgResources = fmt.Sprintf("%d resources", len(checkableResources))
+	}
+
+	if len(policies) > 0 && len(checkableResources) > 0 {
+		fmt.Printf("applying %s to %s... \n", msgPolicies, msgResources)
+	}
+
+	for _, policy := range policies {
+		_, err := policy2.Validate(policy, nil, nil, true, openApiManager)
+		if err != nil {
+			log.Log.Error(err, "skipping invalid policy", "name", policy.GetName())
+			continue
+		}
+
+		matches := common.HasVariables(policy)
+		variable := common.RemoveDuplicateAndObjectVariables(matches)
+
+		if len(variable) > 0 {
+			if len(variables) == 0 {
+				// check policy in variable file
+				if valuesFile == "" || valuesMap[policy.GetName()] == nil {
+					fmt.Printf("test skipped for policy  %v  (as required variables are not provided by the users) \n \n", policy.GetName())
+				}
+			}
+		}
+
+		kindOnwhichPolicyIsApplied := common.GetKindsFromPolicy(policy, subresources, dClient)
+
+		for _, resource := range checkableResources {
+			thisPolicyResourceValues, err := common.CheckVariableForPolicy(valuesMap, globalValMap, policy.GetName(), resource.GetName(), resource.GetKind(), variables, kindOnwhichPolicyIsApplied, variable)
+			if err != nil {
+				return nil, nil, sanitizederror.NewWithError(fmt.Sprintf("policy `%s` have variables. pass the values for the variables for resource `%s` using set/values_file flag", policy.GetName(), resource.GetName()), err)
+			}
+			applyPolicyConfig := common.ApplyPolicyConfig{
+				Policy:                    policy,
+				Resource:                  resource,
+				MutateLogPath:             "",
+				Variables:                 thisPolicyResourceValues,
+				UserInfo:                  userInfo,
+				PolicyReport:              true,
+				NamespaceSelectorMap:      namespaceSelectorMap,
+				Rc:                        &resultCounts,
+				RuleToCloneSourceResource: ruleToCloneSourceResource,
+				Client:                    dClient,
+				Subresources:              subresources,
+			}
+			ers, err := common.ApplyPolicyOnResource(applyPolicyConfig)
+			if err != nil {
+				return nil, nil, sanitizederror.NewWithError(fmt.Errorf("failed to apply policy %v on resource %v", policy.GetName(), resource.GetName()).Error(), err)
+			}
+			engineResponses = append(engineResponses, ers...)
+		}
+	}
+	resultsMap, testResults := buildPolicyResults(engineResponses, values.Results, policyResourcePath, fs, isGit, auditWarn)
+	return resultsMap, testResults, nil
+}
+
+func getFullPath(paths []string, policyResourcePath string, isGit bool) []string {
+	var pols []string
+	var pol string
+	if !isGit {
+		for _, path := range paths {
+			pol = filepath.Join(policyResourcePath, path)
+			pols = append(pols, pol)
+		}
+		return pols
+	}
+	return paths
+}
+
+func selectResourcesForCheck(resources []*unstructured.Unstructured, values *api.Test) []*unstructured.Unstructured {
+	res, _, _ := selectResourcesForCheckInternal(resources, values)
+	return res
+}
+
+// selectResourcesForCheckInternal internal method to test duplicates and unused
+func selectResourcesForCheckInternal(resources []*unstructured.Unstructured, values *api.Test) ([]*unstructured.Unstructured, int, int) {
+	var duplicates int
+	var unused int
+	uniqResources := make(map[string]*unstructured.Unstructured)
+
+	for i := range resources {
+		r := resources[i]
+		key := fmt.Sprintf("%s/%s/%s", r.GetKind(), r.GetName(), r.GetNamespace())
+		if _, ok := uniqResources[key]; ok {
+			fmt.Println("skipping duplicate resource, resource :", r)
+			duplicates++
+		} else {
+			uniqResources[key] = r
+		}
+	}
+
+	selectedResources := map[string]*unstructured.Unstructured{}
+	for key := range uniqResources {
+		r := uniqResources[key]
+		for _, res := range values.Results {
+			if res.Kind == r.GetKind() {
+				for _, testr := range res.Resources {
+					if r.GetName() == testr {
+						selectedResources[key] = r
+					}
+				}
+				if r.GetName() == res.Resource {
+					selectedResources[key] = r
+				}
+			}
+		}
+	}
+
+	var checkableResources []*unstructured.Unstructured
+
+	for key := range selectedResources {
+		checkableResources = append(checkableResources, selectedResources[key])
+		delete(uniqResources, key)
+	}
+	for _, r := range uniqResources {
+		fmt.Println("skipping unused resource, resource :", r)
+		unused++
+	}
+	return checkableResources, duplicates, unused
+}
+
+func buildPolicyResults(
+	engineResponses []engineapi.EngineResponse,
+	testResults []api.TestResults,
+	policyResourcePath string,
+	fs billy.Filesystem,
+	isGit bool,
+	auditWarn bool,
+) (map[string]policyreportv1alpha2.PolicyReportResult, []api.TestResults) {
+	results := map[string]policyreportv1alpha2.PolicyReportResult{}
+
+	for _, resp := range engineResponses {
+		policyName := resp.Policy.GetName()
+		resourceName := resp.Resource.GetName()
+		resourceKind := resp.Resource.GetKind()
+		resourceNamespace := resp.Resource.GetNamespace()
+		policyNamespace := resp.Policy.GetNamespace()
+
+		var rules []string
+		for _, rule := range resp.PolicyResponse.Rules {
+			rules = append(rules, rule.Name())
+		}
+
+		result := policyreportv1alpha2.PolicyReportResult{
+			Policy: policyName,
+			Resources: []corev1.ObjectReference{
+				{
+					Name: resourceName,
+				},
+			},
+			Message: buildMessage(resp),
+		}
+
+		var patchedResourcePath []string
+		for i, test := range testResults {
+			var userDefinedPolicyNamespace string
+			var userDefinedPolicyName string
+			found, err := isNamespacedPolicy(test.Policy)
+			if err != nil {
+				log.Log.V(3).Info("error while checking the policy is namespaced or not", "policy: ", test.Policy, "error: ", err)
+				continue
+			}
+
+			if found {
+				userDefinedPolicyNamespace, userDefinedPolicyName = getUserDefinedPolicyNameAndNamespace(test.Policy)
+				test.Policy = userDefinedPolicyName
+			}
+
+			if test.Resources != nil {
+				if test.Policy == policyName {
+					// results[].namespace value implicit set same as metadata.namespace until and unless
+					// user provides explicit values for results[].namespace in test yaml file.
+					if test.Namespace == "" {
+						test.Namespace = resourceNamespace
+						testResults[i].Namespace = resourceNamespace
+					}
+					for _, resource := range test.Resources {
+						if resource == resourceName {
+							var resultsKey string
+							resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, resource)
+							if !slices.Contains(rules, test.Rule) {
+								if !slices.Contains(rules, "autogen-"+test.Rule) {
+									if !slices.Contains(rules, "autogen-cronjob-"+test.Rule) {
+										result.Result = policyreportv1alpha2.StatusSkip
+									} else {
+										testResults[i].AutoGeneratedRule = "autogen-cronjob"
+										test.Rule = "autogen-cronjob-" + test.Rule
+										resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, resource)
+									}
+								} else {
+									testResults[i].AutoGeneratedRule = "autogen"
+									test.Rule = "autogen-" + test.Rule
+									resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, resource)
+								}
+
+								if results[resultsKey].Result == "" {
+									result.Result = policyreportv1alpha2.StatusSkip
+									results[resultsKey] = result
+								}
+							}
+
+							patchedResourcePath = append(patchedResourcePath, test.PatchedResource)
+							if _, ok := results[resultsKey]; !ok {
+								results[resultsKey] = result
+							}
+						}
+					}
+				}
+			}
+			if test.Resource != "" {
+				if test.Policy == policyName && test.Resource == resourceName {
+					var resultsKey string
+					resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, test.Resource)
+					if !slices.Contains(rules, test.Rule) {
+						if !slices.Contains(rules, "autogen-"+test.Rule) {
+							if !slices.Contains(rules, "autogen-cronjob-"+test.Rule) {
+								result.Result = policyreportv1alpha2.StatusSkip
+							} else {
+								testResults[i].AutoGeneratedRule = "autogen-cronjob"
+								test.Rule = "autogen-cronjob-" + test.Rule
+								resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, test.Resource)
+							}
+						} else {
+							testResults[i].AutoGeneratedRule = "autogen"
+							test.Rule = "autogen-" + test.Rule
+							resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, test.Resource)
+						}
+
+						if results[resultsKey].Result == "" {
+							result.Result = policyreportv1alpha2.StatusSkip
+							results[resultsKey] = result
+						}
+					}
+
+					patchedResourcePath = append(patchedResourcePath, test.PatchedResource)
+					if _, ok := results[resultsKey]; !ok {
+						results[resultsKey] = result
+					}
+				}
+			}
+
+			for _, rule := range resp.PolicyResponse.Rules {
+				if rule.RuleType() != engineapi.Generation || test.Rule != rule.Name() {
+					continue
+				}
+
+				var resultsKey []string
+				var resultKey string
+				var result policyreportv1alpha2.PolicyReportResult
+				resultsKey = GetAllPossibleResultsKey(policyNamespace, policyName, rule.Name(), resourceNamespace, resourceKind, resourceName)
+				for _, key := range resultsKey {
+					if val, ok := results[key]; ok {
+						result = val
+						resultKey = key
+					} else {
+						continue
+					}
+
+					if rule.Status() == engineapi.RuleStatusSkip {
+						result.Result = policyreportv1alpha2.StatusSkip
+					} else if rule.Status() == engineapi.RuleStatusError {
+						result.Result = policyreportv1alpha2.StatusError
+					} else {
+						var x string
+						result.Result = policyreportv1alpha2.StatusFail
+						x = getAndCompareResource(test.GeneratedResource, rule.GeneratedResource(), isGit, policyResourcePath, fs, true)
+						if x == "pass" {
+							result.Result = policyreportv1alpha2.StatusPass
+						}
+					}
+					results[resultKey] = result
+				}
+			}
+
+			for _, rule := range resp.PolicyResponse.Rules {
+				if rule.RuleType() != engineapi.Mutation || test.Rule != rule.Name() {
+					continue
+				}
+
+				var resultsKey []string
+				var resultKey string
+				var result policyreportv1alpha2.PolicyReportResult
+				resultsKey = GetAllPossibleResultsKey(policyNamespace, policyName, rule.Name(), resourceNamespace, resourceKind, resourceName)
+				for _, key := range resultsKey {
+					if val, ok := results[key]; ok {
+						result = val
+						resultKey = key
+					} else {
+						continue
+					}
+
+					if rule.Status() == engineapi.RuleStatusSkip {
+						result.Result = policyreportv1alpha2.StatusSkip
+					} else if rule.Status() == engineapi.RuleStatusError {
+						result.Result = policyreportv1alpha2.StatusError
+					} else {
+						var x string
+						for _, path := range patchedResourcePath {
+							result.Result = policyreportv1alpha2.StatusFail
+							x = getAndCompareResource(path, resp.PatchedResource, isGit, policyResourcePath, fs, false)
+							if x == "pass" {
+								result.Result = policyreportv1alpha2.StatusPass
+								break
+							}
+						}
+					}
+
+					results[resultKey] = result
+				}
+			}
+
+			for _, rule := range resp.PolicyResponse.Rules {
+				if rule.RuleType() != engineapi.Validation && rule.RuleType() != engineapi.ImageVerify || test.Rule != rule.Name() {
+					continue
+				}
+
+				var resultsKey []string
+				var resultKey string
+				var result policyreportv1alpha2.PolicyReportResult
+				resultsKey = GetAllPossibleResultsKey(policyNamespace, policyName, rule.Name(), resourceNamespace, resourceKind, resourceName)
+				for _, key := range resultsKey {
+					if val, ok := results[key]; ok {
+						result = val
+						resultKey = key
+					} else {
+						continue
+					}
+
+					ann := resp.Policy.GetAnnotations()
+					if rule.Status() == engineapi.RuleStatusSkip {
+						result.Result = policyreportv1alpha2.StatusSkip
+					} else if rule.Status() == engineapi.RuleStatusError {
+						result.Result = policyreportv1alpha2.StatusError
+					} else if rule.Status() == engineapi.RuleStatusPass {
+						result.Result = policyreportv1alpha2.StatusPass
+					} else if rule.Status() == engineapi.RuleStatusFail {
+						if scored, ok := ann[kyvernov1.AnnotationPolicyScored]; ok && scored == "false" {
+							result.Result = policyreportv1alpha2.StatusWarn
+						} else if auditWarn && resp.GetValidationFailureAction().Audit() {
+							result.Result = policyreportv1alpha2.StatusWarn
+						} else {
+							result.Result = policyreportv1alpha2.StatusFail
+						}
+					} else {
+						fmt.Println(rule)
+					}
+
+					results[resultKey] = result
+				}
+			}
+		}
+	}
+	return results, testResults
+}
+
+func GetAllPossibleResultsKey(policyNamespace, policy, rule, resourceNamespace, kind, resource string) []string {
+	var resultsKey []string
+	resultKey1 := fmt.Sprintf("%s-%s-%s-%s", policy, rule, kind, resource)
+	resultKey2 := fmt.Sprintf("%s-%s-%s-%s-%s", policy, rule, resourceNamespace, kind, resource)
+	resultKey3 := fmt.Sprintf("%s-%s-%s-%s-%s", policyNamespace, policy, rule, kind, resource)
+	resultKey4 := fmt.Sprintf("%s-%s-%s-%s-%s-%s", policyNamespace, policy, rule, resourceNamespace, kind, resource)
+	resultsKey = append(resultsKey, resultKey1, resultKey2, resultKey3, resultKey4)
+	return resultsKey
+}
+
+func GetResultKeyAccordingToTestResults(policyNs, policy, rule, resourceNs, kind, resource string) string {
+	var resultKey string
+	resultKey = fmt.Sprintf("%s-%s-%s-%s", policy, rule, kind, resource)
+
+	if policyNs != "" && resourceNs != "" {
+		resultKey = fmt.Sprintf("%s-%s-%s-%s-%s-%s", policyNs, policy, rule, resourceNs, kind, resource)
+	} else if policyNs != "" {
+		resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", policyNs, policy, rule, kind, resource)
+	} else if resourceNs != "" {
+		resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", policy, rule, resourceNs, kind, resource)
+	}
+	return resultKey
+}
+
+func isNamespacedPolicy(policyNames string) (bool, error) {
+	return regexp.MatchString("^[a-z]*/[a-z]*", policyNames)
+}
+
+// getAndCompareResource --> Get the patchedResource or generatedResource from the path provided by user
+// And compare this resource with engine generated resource.
+func getAndCompareResource(path string, engineResource unstructured.Unstructured, isGit bool, policyResourcePath string, fs billy.Filesystem, isGenerate bool) string {
+	var status string
+	resourceType := "patchedResource"
+	if isGenerate {
+		resourceType = "generatedResource"
+	}
+
+	userResource, err := common.GetResourceFromPath(fs, path, isGit, policyResourcePath, resourceType)
+	if err != nil {
+		fmt.Printf("Error: failed to load resources\nCause: %s\n", err)
+		return ""
+	}
+	matched, err := generate.ValidateResourceWithPattern(log.Log, engineResource.UnstructuredContent(), userResource.UnstructuredContent())
+	if err != nil {
+		log.Log.V(3).Info(resourceType+" mismatch", "error", err.Error())
+		status = "fail"
+	} else if matched == "" {
+		status = "pass"
+	}
+	return status
+}
+
+func buildMessage(resp engineapi.EngineResponse) string {
+	var messages []string
+	for _, ruleResp := range resp.PolicyResponse.Rules {
+		message := strings.TrimSpace(ruleResp.Message())
+		if message != "" {
+			messages = append(messages, message)
+		}
+	}
+	return strings.Join(messages, ",")
+}
+
+func getUserDefinedPolicyNameAndNamespace(policyName string) (string, string) {
+	if strings.Contains(policyName, "/") {
+		parts := strings.Split(policyName, "/")
+		namespace := parts[0]
+		policy := parts[1]
+		return namespace, policy
+	}
+	return "", policyName
+}

cmd/cli/kubectl-kyverno/utils/common/common.go

@@ -25,6 +25,7 @@ import (
 	engineContext "github.com/kyverno/kyverno/pkg/engine/context"
 	"github.com/kyverno/kyverno/pkg/engine/jmespath"
 	"github.com/kyverno/kyverno/pkg/engine/variables/regex"
+	"github.com/kyverno/kyverno/pkg/logging"
 	"github.com/kyverno/kyverno/pkg/registryclient"
 	datautils "github.com/kyverno/kyverno/pkg/utils/data"
 	kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
@@ -35,9 +36,10 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/yaml"
-	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
+var log = logging.WithName("kubectl-kyverno")
+
 type ResultCounts struct {
 	Pass  int
 	Fail  int
@@ -107,7 +109,7 @@ func HasVariables(policy kyvernov1.PolicyInterface) [][]string {
 // GetPolicies - Extracting the policies from multiple YAML
 func GetPolicies(paths []string) (policies []kyvernov1.PolicyInterface, errors []error) {
 	for _, path := range paths {
-		log.Log.V(5).Info("reading policies", "path", path)
+		log.V(5).Info("reading policies", "path", path)
 
 		var (
 			fileDesc os.FileInfo
@@ -199,7 +201,7 @@ func GetPolicies(paths []string) (policies []kyvernov1.PolicyInterface, errors [
 		}
 	}
 
-	log.Log.V(3).Info("read policies", "policies", len(policies), "errors", len(errors))
+	log.V(3).Info("read policies", "policies", len(policies), "errors", len(errors))
 	return policies, errors
 }
 
@@ -286,12 +288,12 @@ func GetVariable(variablesString, valuesFile string, fs billy.Filesystem, isGit
 		if values.GlobalValues == nil {
 			values.GlobalValues = make(map[string]string)
 			values.GlobalValues["request.operation"] = "CREATE"
-			log.Log.V(3).Info("Defaulting request.operation to CREATE")
+			log.V(3).Info("Defaulting request.operation to CREATE")
 		} else {
 			if val, ok := values.GlobalValues["request.operation"]; ok {
 				if val == "" {
 					values.GlobalValues["request.operation"] = "CREATE"
-					log.Log.V(3).Info("Globally request.operation value provided by the user is empty, defaulting it to CREATE", "request.opearation: ", values.GlobalValues)
+					log.V(3).Info("Globally request.operation value provided by the user is empty, defaulting it to CREATE", "request.opearation: ", values.GlobalValues)
 				}
 			}
 		}
@@ -304,7 +306,7 @@ func GetVariable(variablesString, valuesFile string, fs billy.Filesystem, isGit
 				if val, ok := r.Values["request.operation"]; ok {
 					if val == "" {
 						r.Values["request.operation"] = "CREATE"
-						log.Log.V(3).Info("No request.operation found, defaulting it to CREATE", "policy", p.Name)
+						log.V(3).Info("No request.operation found, defaulting it to CREATE", "policy", p.Name)
 					}
 				}
 				for variableInFile := range r.Values {
@@ -343,7 +345,7 @@ func GetVariable(variablesString, valuesFile string, fs billy.Filesystem, isGit
 	if globalValMap != nil {
 		if _, ok := globalValMap["request.operation"]; !ok {
 			globalValMap["request.operation"] = "CREATE"
-			log.Log.V(3).Info("Defaulting request.operation to CREATE")
+			log.V(3).Info("Defaulting request.operation to CREATE")
 		}
 	}
 
@@ -423,16 +425,16 @@ OuterLoop:
 	}
 
 	resPath := fmt.Sprintf("%s/%s/%s", c.Resource.GetNamespace(), c.Resource.GetKind(), c.Resource.GetName())
-	log.Log.V(3).Info("applying policy on resource", "policy", c.Policy.GetName(), "resource", resPath)
+	log.V(3).Info("applying policy on resource", "policy", c.Policy.GetName(), "resource", resPath)
 
 	resourceRaw, err := c.Resource.MarshalJSON()
 	if err != nil {
-		log.Log.Error(err, "failed to marshal resource")
+		log.Error(err, "failed to marshal resource")
 	}
 
 	updatedResource, err := kubeutils.BytesToUnstructured(resourceRaw)
 	if err != nil {
-		log.Log.Error(err, "unable to convert raw resource to unstructured")
+		log.Error(err, "unable to convert raw resource to unstructured")
 	}
 	ctx := engineContext.NewContext(jp)
 
@@ -443,19 +445,19 @@ OuterLoop:
 	}
 
 	if err != nil {
-		log.Log.Error(err, "failed to load resource in context")
+		log.Error(err, "failed to load resource in context")
 	}
 
 	for key, value := range c.Variables {
 		err = ctx.AddVariable(key, value)
 		if err != nil {
-			log.Log.Error(err, "failed to add variable to context")
+			log.Error(err, "failed to add variable to context")
 		}
 	}
 
 	cfg := config.NewDefaultConfiguration(false)
 	if err := ctx.AddImageInfos(c.Resource, cfg); err != nil {
-		log.Log.Error(err, "failed to add image variables to context")
+		log.Error(err, "failed to add image variables to context")
 	}
 
 	gvk, subresource := updatedResource.GroupVersionKind(), ""
@@ -541,7 +543,7 @@ OuterLoop:
 		if !generateResponse.IsEmpty() {
 			newRuleResponse, err := handleGeneratePolicy(&generateResponse, *policyContext, c.RuleToCloneSourceResource)
 			if err != nil {
-				log.Log.Error(err, "failed to apply generate policy")
+				log.Error(err, "failed to apply generate policy")
 			} else {
 				generateResponse.PolicyResponse.Rules = newRuleResponse
 			}
@@ -627,7 +629,7 @@ func PrintMutatedOutput(mutateLogPath string, mutateLogPathIsDir bool, yaml stri
 	if _, err := f.Write([]byte(yaml)); err != nil {
 		closeErr := f.Close()
 		if closeErr != nil {
-			log.Log.Error(closeErr, "failed to close file")
+			log.Error(closeErr, "failed to close file")
 		}
 		return err
 	}
@@ -687,7 +689,7 @@ func GetPoliciesFromPaths(fs billy.Filesystem, dirPath []string, isGit bool, pol
 				}
 				return nil, sanitizederror.New(fmt.Sprintf("no file found in paths %v", dirPath))
 			}
-			if len(errors) > 0 && log.Log.V(1).Enabled() {
+			if len(errors) > 0 && log.V(1).Enabled() {
 				fmt.Printf("ignoring errors: \n")
 				for _, e := range errors {
 					fmt.Printf("    %v \n", e.Error())
@@ -893,7 +895,7 @@ func PrintMutatedPolicy(mutatedPolicies []kyvernov1.PolicyInterface) error {
 		if err != nil {
 			return sanitizederror.NewWithError("failed to marsal mutated policy", err)
 		}
-		log.Log.V(5).Info("mutated Policy:", string(p))
+		log.V(5).Info("mutated Policy:", string(p))
 	}
 	return nil
 }
@@ -1087,7 +1089,7 @@ func handleGeneratePolicy(generateResponse *engineapi.EngineResponse, policyCont
 	var newRuleResponse []engineapi.RuleResponse
 
 	for _, rule := range generateResponse.PolicyResponse.Rules {
-		genResource, err := c.ApplyGeneratePolicy(log.Log, &policyContext, gr, []string{rule.Name()})
+		genResource, err := c.ApplyGeneratePolicy(log.V(2), &policyContext, gr, []string{rule.Name()})
 		if err != nil {
 			return nil, err
 		}
@@ -1137,7 +1139,7 @@ func GetUserInfoFromPath(fs billy.Filesystem, path string, isGit bool, policyRes
 		if err := json.Unmarshal(userInfoBytes, userInfo); err != nil {
 			errors = append(errors, sanitizederror.NewWithError("failed to decode yaml", err))
 		}
-		if len(errors) > 0 && log.Log.V(1).Enabled() {
+		if len(errors) > 0 && log.V(1).Enabled() {
 			fmt.Printf("ignoring errors: \n")
 			for _, e := range errors {
 				fmt.Printf("    %v \n", e.Error())

cmd/cli/kubectl-kyverno/utils/common/fetch.go

@@ -20,7 +20,6 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes/scheme"
-	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/yaml"
 )
 
@@ -88,7 +87,7 @@ func whenClusterIsTrue(resourceTypes []schema.GroupVersionKind, subresourceMap m
 			}
 			if lenOfResource >= len(resources) {
 				if policyReport {
-					log.Log.V(3).Info(fmt.Sprintf("%s not found in cluster", resourcePath))
+					log.V(3).Info(fmt.Sprintf("%s not found in cluster", resourcePath))
 				} else {
 					fmt.Printf("\n----------------------------------------------------------------------\nresource %s not found in cluster\n----------------------------------------------------------------------\n", resourcePath)
 				}
@@ -105,7 +104,7 @@ func whenClusterIsFalse(resourcePaths []string, policyReport bool) ([]*unstructu
 		resourceBytes, err := getFileBytes(resourcePath)
 		if err != nil {
 			if policyReport {
-				log.Log.V(3).Info(fmt.Sprintf("failed to load resources: %s.", resourcePath), "error", err)
+				log.V(3).Info(fmt.Sprintf("failed to load resources: %s.", resourcePath), "error", err)
 			} else {
 				fmt.Printf("\n----------------------------------------------------------------------\nfailed to load resources: %s. \nerror: %s\n----------------------------------------------------------------------\n", resourcePath, err)
 			}
@@ -177,7 +176,7 @@ func GetResource(resourceBytes []byte) ([]*unstructured.Unstructured, error) {
 		resource, err := convertResourceToUnstructured(resourceYaml)
 		if err != nil {
 			if strings.Contains(err.Error(), "Object 'Kind' is missing") {
-				log.Log.V(3).Info("skipping resource as kind not found")
+				log.V(3).Info("skipping resource as kind not found")
 				continue
 			}
 			getErrString = getErrString + err.Error() + "\n"
@@ -345,7 +344,7 @@ func addGVKToResourceTypesMap(kind string, resourceTypesMap map[schema.GroupVers
 	group, version, kind, subresource := kubeutils.ParseKindSelector(kind)
 	gvrss, err := client.Discovery().FindResources(group, version, kind, subresource)
 	if err != nil {
-		log.Log.Info("failed to find resource", "kind", kind, "error", err)
+		log.Info("failed to find resource", "kind", kind, "error", err)
 		return
 	}
 	for parent, child := range gvrss {