Fix cli load policies from fs (#10270)

* skip invalid policy files

Signed-off-by: Jim Bugwadia <jim@nirmata.com>

* fix file-system policy loader

Signed-off-by: Jim Bugwadia <jim@nirmata.com>

* propagate policy schema error

Signed-off-by: Jim Bugwadia <jim@nirmata.com>

---------

Signed-off-by: Jim Bugwadia <jim@nirmata.com>

cmd/cli/kubectl-kyverno/_testdata/policies-invalid/artifacthub-pkg.yml

@@ -0,0 +1,22 @@
+name: add-network-policy
+version: 1.0.0
+displayName: Add Network Policy
+createdAt: "2023-04-10T19:47:15.000Z"
+description: >-
+  By default, Kubernetes allows communications across all Pods within a cluster. The NetworkPolicy resource and a CNI plug-in that supports NetworkPolicy must be used to restrict communications. A default NetworkPolicy should be configured for each Namespace to default deny all ingress and egress traffic to the Pods in the Namespace. Application teams can then configure additional NetworkPolicy resources to allow desired traffic to application Pods from select sources. This policy will create a new NetworkPolicy resource named `default-deny` which will deny all traffic anytime a new Namespace is created.
+install: |-
+  ```shell
+  kubectl apply -f https://raw.githubusercontent.com/kyverno/policies/main/best-practices/add-network-policy/add-network-policy.yaml
+  ```
+keywords:
+  - kyverno
+  - Multi-Tenancy
+  - EKS Best Practices
+readme: |
+  By default, Kubernetes allows communications across all Pods within a cluster. The NetworkPolicy resource and a CNI plug-in that supports NetworkPolicy must be used to restrict communications. A default NetworkPolicy should be configured for each Namespace to default deny all ingress and egress traffic to the Pods in the Namespace. Application teams can then configure additional NetworkPolicy resources to allow desired traffic to application Pods from select sources. This policy will create a new NetworkPolicy resource named `default-deny` which will deny all traffic anytime a new Namespace is created.
+  
+  Refer to the documentation for more details on Kyverno annotations: https://artifacthub.io/docs/topics/annotations/kyverno/
+annotations:
+  kyverno/category: "Multi-Tenancy, EKS Best Practices"
+  kyverno/subject: "NetworkPolicy"
+digest: d01c7f24cf053549534bba5b98cc479ee0e5a4a01f810b8a45d11c86b26d846e

cmd/cli/kubectl-kyverno/_testdata/policies-invalid/chainsaw-test.yaml

@@ -0,0 +1,27 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: add-network-policy
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: old-resource.yaml
+  - name: step-02
+    try:
+    - apply:
+        file: ../add-network-policy.yaml
+    - assert:
+        file: policy-ready.yaml
+  - name: step-03
+    try:
+    - apply:
+        file: ../.kyverno-test/resource.yaml
+  - name: step-04
+    try:
+    - assert:
+        file: ../.kyverno-test/generatedResource.yaml
+    - error:
+        file: notGeneratedResource.yaml

cmd/cli/kubectl-kyverno/_testdata/policies-invalid/policy-ready.yaml

@@ -0,0 +1,6 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: add-networkpolicy
+status:
+  ready: true

cmd/cli/kubectl-kyverno/_testdata/policies-mixed/artifacthub-pkg.yml

@@ -0,0 +1,22 @@
+name: add-network-policy
+version: 1.0.0
+displayName: Add Network Policy
+createdAt: "2023-04-10T19:47:15.000Z"
+description: >-
+  By default, Kubernetes allows communications across all Pods within a cluster. The NetworkPolicy resource and a CNI plug-in that supports NetworkPolicy must be used to restrict communications. A default NetworkPolicy should be configured for each Namespace to default deny all ingress and egress traffic to the Pods in the Namespace. Application teams can then configure additional NetworkPolicy resources to allow desired traffic to application Pods from select sources. This policy will create a new NetworkPolicy resource named `default-deny` which will deny all traffic anytime a new Namespace is created.
+install: |-
+  ```shell
+  kubectl apply -f https://raw.githubusercontent.com/kyverno/policies/main/best-practices/add-network-policy/add-network-policy.yaml
+  ```
+keywords:
+  - kyverno
+  - Multi-Tenancy
+  - EKS Best Practices
+readme: |
+  By default, Kubernetes allows communications across all Pods within a cluster. The NetworkPolicy resource and a CNI plug-in that supports NetworkPolicy must be used to restrict communications. A default NetworkPolicy should be configured for each Namespace to default deny all ingress and egress traffic to the Pods in the Namespace. Application teams can then configure additional NetworkPolicy resources to allow desired traffic to application Pods from select sources. This policy will create a new NetworkPolicy resource named `default-deny` which will deny all traffic anytime a new Namespace is created.
+  
+  Refer to the documentation for more details on Kyverno annotations: https://artifacthub.io/docs/topics/annotations/kyverno/
+annotations:
+  kyverno/category: "Multi-Tenancy, EKS Best Practices"
+  kyverno/subject: "NetworkPolicy"
+digest: d01c7f24cf053549534bba5b98cc479ee0e5a4a01f810b8a45d11c86b26d846e

cmd/cli/kubectl-kyverno/_testdata/policies-mixed/cpol-pod-requirements.yaml

@@ -0,0 +1,44 @@
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+    policies.kyverno.io/category: Pod Security Standards (Restricted)
+    policies.kyverno.io/severity: medium
+  name: pod-requirements
+spec:
+  admission: true
+  background: false
+  rules:
+  - match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    name: pods-require-account
+    validate:
+      message: User pods must include an account for charging
+      pattern:
+        metadata:
+          labels:
+            account: '*?'
+  - match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    name: pods-require-limits
+    validate:
+      message: CPU and memory resource requests and limits are required for user pods
+      pattern:
+        spec:
+          containers:
+          - resources:
+              limits:
+                cpu: ?*
+                memory: ?*
+              requests:
+                cpu: ?*
+                memory: ?*
+  validationFailureAction: Audit

cmd/cli/kubectl-kyverno/_testdata/policies-mixed/nested/artifacthub-pkg.yml

@@ -0,0 +1,22 @@
+name: add-network-policy
+version: 1.0.0
+displayName: Add Network Policy
+createdAt: "2023-04-10T19:47:15.000Z"
+description: >-
+  By default, Kubernetes allows communications across all Pods within a cluster. The NetworkPolicy resource and a CNI plug-in that supports NetworkPolicy must be used to restrict communications. A default NetworkPolicy should be configured for each Namespace to default deny all ingress and egress traffic to the Pods in the Namespace. Application teams can then configure additional NetworkPolicy resources to allow desired traffic to application Pods from select sources. This policy will create a new NetworkPolicy resource named `default-deny` which will deny all traffic anytime a new Namespace is created.
+install: |-
+  ```shell
+  kubectl apply -f https://raw.githubusercontent.com/kyverno/policies/main/best-practices/add-network-policy/add-network-policy.yaml
+  ```
+keywords:
+  - kyverno
+  - Multi-Tenancy
+  - EKS Best Practices
+readme: |
+  By default, Kubernetes allows communications across all Pods within a cluster. The NetworkPolicy resource and a CNI plug-in that supports NetworkPolicy must be used to restrict communications. A default NetworkPolicy should be configured for each Namespace to default deny all ingress and egress traffic to the Pods in the Namespace. Application teams can then configure additional NetworkPolicy resources to allow desired traffic to application Pods from select sources. This policy will create a new NetworkPolicy resource named `default-deny` which will deny all traffic anytime a new Namespace is created.
+  
+  Refer to the documentation for more details on Kyverno annotations: https://artifacthub.io/docs/topics/annotations/kyverno/
+annotations:
+  kyverno/category: "Multi-Tenancy, EKS Best Practices"
+  kyverno/subject: "NetworkPolicy"
+digest: d01c7f24cf053549534bba5b98cc479ee0e5a4a01f810b8a45d11c86b26d846e

cmd/cli/kubectl-kyverno/_testdata/policies-mixed/nested/cpol-pod-requirements.yaml

@@ -0,0 +1,44 @@
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+    policies.kyverno.io/category: Pod Security Standards (Restricted)
+    policies.kyverno.io/severity: medium
+  name: pod-requirements
+spec:
+  admission: true
+  background: false
+  rules:
+  - match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    name: pods-require-account
+    validate:
+      message: User pods must include an account for charging
+      pattern:
+        metadata:
+          labels:
+            account: '*?'
+  - match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    name: pods-require-limits
+    validate:
+      message: CPU and memory resource requests and limits are required for user pods
+      pattern:
+        spec:
+          containers:
+          - resources:
+              limits:
+                cpu: ?*
+                memory: ?*
+              requests:
+                cpu: ?*
+                memory: ?*
+  validationFailureAction: Audit

cmd/cli/kubectl-kyverno/commands/apply/command.go

@@ -355,22 +355,23 @@ func (c *ApplyCommandConfig) loadPolicies(skipInvalidPolicies SkippedInvalidPoli
 				return nil, nil, skipInvalidPolicies, nil, nil, nil, nil, fmt.Errorf("failed to list YAMLs in repository (%w)", err)
 			}
 			for _, policyYaml := range policyYamls {
-				policiesFromFile, vapsFromFile, vapBindingsFromFile, err := policy.Load(fs, "", policyYaml)
+				loaderResults, err := policy.Load(fs, "", policyYaml)
 				if err != nil {
 					continue
 				}
-				policies = append(policies, policiesFromFile...)
+				policies = append(policies, loaderResults.Policies...)
-				vaps = append(vaps, vapsFromFile...)
+				vaps = append(vaps, loaderResults.VAPs...)
-				vapBindings = append(vapBindings, vapBindingsFromFile...)
+				vapBindings = append(vapBindings, loaderResults.VAPBindings...)
 			}
 		} else {
-			policiesFromFile, vapsFromFile, vapBindingsFromFile, err := policy.Load(nil, "", path)
+			loaderResults, err := policy.Load(nil, "", path)
 			if err != nil {
-				return nil, nil, skipInvalidPolicies, nil, nil, nil, nil, fmt.Errorf("failed to load policies (%w)", err)
+				log.Log.V(3).Info("skipping invalid YAML file", "path", path, "error", err)
+			} else {
+				policies = append(policies, loaderResults.Policies...)
+				vaps = append(vaps, loaderResults.VAPs...)
+				vapBindings = append(vapBindings, loaderResults.VAPBindings...)
 			}
-			policies = append(policies, policiesFromFile...)
-			vaps = append(vaps, vapsFromFile...)
-			vapBindings = append(vapBindings, vapBindingsFromFile...)
 		}
 	}
 

cmd/cli/kubectl-kyverno/commands/fix/policy/options.go

@@ -61,15 +61,15 @@ func (o options) execute(out io.Writer, dirs ...string) error {
 }
 
 func (o options) processFile(out io.Writer, path string) {
-	policies, vaps, vapBindings, err := policy.LoadWithLoader(policy.KubectlValidateLoader, nil, "", path)
+	results, err := policy.LoadWithLoader(policy.KubectlValidateLoader, nil, "", path)
 	if err != nil {
 		return
 	}
-	if len(policies) == 0 {
+	if results == nil || len(results.Policies) == 0 {
 		return
 	}
-	fixed := make([]kyvernov1.PolicyInterface, 0, len(policies))
+	fixed := make([]kyvernov1.PolicyInterface, 0, len(results.Policies))
-	for _, policy := range policies {
+	for _, policy := range results.Policies {
 		copy := policy.CreateDeepCopy()
 		fmt.Fprintf(out, "Processing file (%s)...\n", path)
 		messages, err := fix.FixPolicy(copy)
@@ -82,7 +82,7 @@ func (o options) processFile(out io.Writer, path string) {
 		}
 		fixed = append(fixed, copy)
 	}
-	needsSave := !reflect.DeepEqual(policies, fixed)
+	needsSave := !reflect.DeepEqual(results.Policies, fixed)
 	if o.save && needsSave {
 		fmt.Fprintf(out, "  Saving file (%s)...", path)
 		fmt.Fprintln(out)
@@ -159,7 +159,7 @@ func (o options) processFile(out io.Writer, path string) {
 			yamlBytes = append(yamlBytes, []byte("---\n")...)
 			yamlBytes = append(yamlBytes, finalBytes...)
 		}
-		for _, vap := range vaps {
+		for _, vap := range results.VAPs {
 			finalBytes, err := yaml.Marshal(vap)
 			if err != nil {
 				fmt.Fprintf(out, "    ERROR: converting to yaml: %s", err)
@@ -169,7 +169,7 @@ func (o options) processFile(out io.Writer, path string) {
 			yamlBytes = append(yamlBytes, []byte("---\n")...)
 			yamlBytes = append(yamlBytes, finalBytes...)
 		}
-		for _, vapBinding := range vapBindings {
+		for _, vapBinding := range results.VAPBindings {
 			finalBytes, err := yaml.Marshal(vapBinding)
 			if err != nil {
 				fmt.Fprintf(out, "    ERROR: converting to yaml: %s", err)

cmd/cli/kubectl-kyverno/commands/oci/push/options.go

@@ -35,11 +35,11 @@ func (o options) validate(policy string) error {
 }
 
 func (o options) execute(ctx context.Context, dir string, keychain authn.Keychain) error {
-	policies, _, _, err := policy.Load(nil, "", dir)
+	results, err := policy.Load(nil, "", dir)
 	if err != nil {
 		return fmt.Errorf("unable to read policy file or directory %s (%w)", dir, err)
 	}
-	for _, policy := range policies {
+	for _, policy := range results.Policies {
 		if _, err := policyvalidation.Validate(policy, nil, nil, nil, true, config.KyvernoUserName(config.KyvernoServiceAccountName())); err != nil {
 			return fmt.Errorf("validating policy %s: %v", policy.GetName(), err)
 		}
@@ -50,7 +50,7 @@ func (o options) execute(ctx context.Context, dir string, keychain authn.Keychai
 	if err != nil {
 		return fmt.Errorf("parsing image reference: %v", err)
 	}
-	for _, policy := range policies {
+	for _, policy := range results.Policies {
 		if policy.IsNamespaced() {
 			fmt.Fprintf(os.Stderr, "Adding policy [%s]\n", policy.GetName())
 		} else {

cmd/cli/kubectl-kyverno/commands/test/command.go

@@ -116,16 +116,16 @@ func testCommandExecute(
 				continue
 			}
 			resourcePath := filepath.Dir(test.Path)
-			responses, err := runTest(out, test, registryAccess, false)
+			responses, err := runTest(out, test, registryAccess)
 			if err != nil {
 				return fmt.Errorf("failed to run test (%w)", err)
 			}
 			fmt.Fprintln(out, "  Checking results ...")
 			var resultsTable table.Table
-			if err := printTestResult(out, filteredResults, responses, rc, &resultsTable, test.Fs, resourcePath); err != nil {
+			if err := printTestResult(filteredResults, responses, rc, &resultsTable, test.Fs, resourcePath); err != nil {
 				return fmt.Errorf("failed to print test result (%w)", err)
 			}
-			if err := printCheckResult(out, test.Test.Checks, responses, rc, &resultsTable); err != nil {
+			if err := printCheckResult(test.Test.Checks, responses, rc, &resultsTable); err != nil {
 				return fmt.Errorf("failed to print test result (%w)", err)
 			}
 			fullTable.AddFailed(resultsTable.RawRows...)

cmd/cli/kubectl-kyverno/commands/test/compare.go

@@ -11,13 +11,13 @@ import (
 func getAndCompareResource(actualResource unstructured.Unstructured, fs billy.Filesystem, path string) (bool, error) {
 	expectedResource, err := resource.GetResourceFromPath(fs, path)
 	if err != nil {
-		return false, fmt.Errorf("Error: failed to load resource (%s)", err)
+		return false, fmt.Errorf("error: failed to load resource (%s)", err)
 	}
 	resource.FixupGenerateLabels(actualResource)
 	resource.FixupGenerateLabels(*expectedResource)
 	equals, err := resource.Compare(actualResource, *expectedResource, true)
 	if err != nil {
-		return false, fmt.Errorf("Error: failed to compare resources (%s)", err)
+		return false, fmt.Errorf("error: failed to compare resources (%s)", err)
 	}
 	return equals, nil
 }

cmd/cli/kubectl-kyverno/commands/test/load.go

@@ -60,11 +60,11 @@ func loadTest(path string, fileName string, gitBranch string) (test.TestCases, e
 				}
 			}
 			if _, err := gitutils.Clone(repoURL, fs, gitBranch); err != nil {
-				return nil, fmt.Errorf("Error: failed to clone repository \nCause: %s\n", err)
+				return nil, fmt.Errorf("error: failed to clone repository \nCause: %s", err)
 			}
 			yamlFiles, err := gitutils.ListYamls(fs, gitPathToYamls)
 			if err != nil {
-				return nil, fmt.Errorf("failed to list YAMLs in repository (%w)", err)
+				return nil, fmt.Errorf("error: failed to list YAMLs in repository (%w)", err)
 			}
 			sort.Strings(yamlFiles)
 			for _, yamlFilePath := range yamlFiles {

cmd/cli/kubectl-kyverno/commands/test/output.go

@@ -16,7 +16,6 @@ import (
 )
 
 func printCheckResult(
-	out io.Writer,
 	checks []v1alpha1.CheckResult,
 	responses []engineapi.EngineResponse,
 	rc *resultCounts,
@@ -162,7 +161,6 @@ func printCheckResult(
 }
 
 func printTestResult(
-	out io.Writer,
 	tests []v1alpha1.TestResult,
 	responses []engineapi.EngineResponse,
 	rc *resultCounts,

cmd/cli/kubectl-kyverno/commands/test/test.go

@@ -28,7 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
 
-func runTest(out io.Writer, testCase test.TestCase, registryAccess bool, auditWarn bool) ([]engineapi.EngineResponse, error) {
+func runTest(out io.Writer, testCase test.TestCase, registryAccess bool) ([]engineapi.EngineResponse, error) {
 	// don't process test case with errors
 	if testCase.Err != nil {
 		return nil, testCase.Err
@@ -50,7 +50,7 @@ func runTest(out io.Writer, testCase test.TestCase, registryAccess bool, auditWa
 		fmt.Fprintln(out, "  Loading user infos", "...")
 		info, err := userinfo.Load(testCase.Fs, testCase.Test.UserInfo, testDir)
 		if err != nil {
-			return nil, fmt.Errorf("Error: failed to load request info (%s)", err)
+			return nil, fmt.Errorf("error: failed to load request info (%s)", err)
 		}
 		deprecations.CheckUserInfo(out, testCase.Test.UserInfo, info)
 		userInfo = &info.RequestInfo
@@ -58,21 +58,21 @@ func runTest(out io.Writer, testCase test.TestCase, registryAccess bool, auditWa
 	// policies
 	fmt.Fprintln(out, "  Loading policies", "...")
 	policyFullPath := path.GetFullPaths(testCase.Test.Policies, testDir, isGit)
-	policies, vaps, vapBindings, err := policy.Load(testCase.Fs, testDir, policyFullPath...)
+	results, err := policy.Load(testCase.Fs, testDir, policyFullPath...)
 	if err != nil {
-		return nil, fmt.Errorf("Error: failed to load policies (%s)", err)
+		return nil, fmt.Errorf("error: failed to load policies (%s)", err)
 	}
 	// resources
 	fmt.Fprintln(out, "  Loading resources", "...")
 	resourceFullPath := path.GetFullPaths(testCase.Test.Resources, testDir, isGit)
-	resources, err := common.GetResourceAccordingToResourcePath(out, testCase.Fs, resourceFullPath, false, policies, vaps, dClient, "", false, testDir)
+	resources, err := common.GetResourceAccordingToResourcePath(out, testCase.Fs, resourceFullPath, false, results.Policies, results.VAPs, dClient, "", false, testDir)
 	if err != nil {
-		return nil, fmt.Errorf("Error: failed to load resources (%s)", err)
+		return nil, fmt.Errorf("error: failed to load resources (%s)", err)
 	}
 	uniques, duplicates := resource.RemoveDuplicates(resources)
 	if len(duplicates) > 0 {
 		for dup := range duplicates {
-			fmt.Fprintln(out, "  Warning: found duplicated resource", dup.Kind, dup.Name, dup.Namespace)
+			fmt.Fprintln(out, "  warning: found duplicated resource", dup.Kind, dup.Name, dup.Namespace)
 		}
 	}
 	// exceptions
@@ -80,11 +80,11 @@ func runTest(out io.Writer, testCase test.TestCase, registryAccess bool, auditWa
 	exceptionFullPath := path.GetFullPaths(testCase.Test.PolicyExceptions, testDir, isGit)
 	exceptions, err := exception.Load(exceptionFullPath...)
 	if err != nil {
-		return nil, fmt.Errorf("Error: failed to load exceptions (%s)", err)
+		return nil, fmt.Errorf("error: failed to load exceptions (%s)", err)
 	}
 	// Validates that exceptions cannot be used with ValidatingAdmissionPolicies.
-	if len(vaps) > 0 && len(exceptions) > 0 {
+	if len(results.VAPs) > 0 && len(exceptions) > 0 {
-		return nil, fmt.Errorf("Error: Currently, the use of exceptions in conjunction with ValidatingAdmissionPolicies is not supported.")
+		return nil, fmt.Errorf("error: use of exceptions with ValidatingAdmissionPolicies is not supported")
 	}
 	// init store
 	var store store.Store
@@ -93,14 +93,22 @@ func runTest(out io.Writer, testCase test.TestCase, registryAccess bool, auditWa
 	if vars != nil {
 		vars.SetInStore(&store)
 	}
+
+	policyCount := len(results.Policies) + len(results.VAPs)
+	policyPlural := pluralize.Pluralize(len(results.Policies)+len(results.VAPs), "policy", "policies")
+	resourceCount := len(uniques)
+	resourcePlural := pluralize.Pluralize(len(uniques), "resource", "resources")
 	if len(exceptions) > 0 {
-		fmt.Fprintln(out, "  Applying", len(policies)+len(vaps), pluralize.Pluralize(len(policies)+len(vaps), "policy", "policies"), "to", len(uniques), pluralize.Pluralize(len(uniques), "resource", "resources"), "with", len(exceptions), pluralize.Pluralize(len(exceptions), "exception", "exceptions"), "...")
+		exceptionCount := len(exceptions)
+		exceptionsPlural := pluralize.Pluralize(len(exceptions), "exception", "exceptions")
+		fmt.Fprintln(out, "  Applying", policyCount, policyPlural, "to", resourceCount, resourcePlural, "with", exceptionCount, exceptionsPlural, "...")
 	} else {
-		fmt.Fprintln(out, "  Applying", len(policies)+len(vaps), pluralize.Pluralize(len(policies)+len(vaps), "policy", "policies"), "to", len(uniques), pluralize.Pluralize(len(uniques), "resource", "resources"), "...")
+		fmt.Fprintln(out, "  Applying", policyCount, policyPlural, "to", resourceCount, resourcePlural, "...")
 	}
+
 	// TODO document the code below
 	ruleToCloneSourceResource := map[string]string{}
-	for _, policy := range policies {
+	for _, policy := range results.Policies {
 		for _, rule := range autogen.ComputeRules(policy, "") {
 			for _, res := range testCase.Test.Results {
 				if res.IsValidatingAdmissionPolicy {
@@ -142,8 +150,8 @@ func runTest(out io.Writer, testCase test.TestCase, registryAccess bool, auditWa
 		}
 	}
 	// validate policies
-	validPolicies := make([]kyvernov1.PolicyInterface, 0, len(policies))
+	validPolicies := make([]kyvernov1.PolicyInterface, 0, len(results.Policies))
-	for _, pol := range policies {
+	for _, pol := range results.Policies {
 		// TODO we should return this info to the caller
 		_, err := policyvalidation.Validate(pol, nil, nil, nil, true, config.KyvernoUserName(config.KyvernoServiceAccountName()))
 		if err != nil {
@@ -180,8 +188,8 @@ func runTest(out io.Writer, testCase test.TestCase, registryAccess bool, auditWa
 	}
 	for _, resource := range uniques {
 		processor := processor.ValidatingAdmissionPolicyProcessor{
-			Policies:             vaps,
+			Policies:             results.VAPs,
-			Bindings:             vapBindings,
+			Bindings:             results.VAPBindings,
 			Resource:             resource,
 			NamespaceSelectorMap: vars.NamespaceSelectors(),
 			PolicyReport:         true,

cmd/cli/kubectl-kyverno/policy/legacy_loader.go

@@ -0,0 +1,16 @@
+package policy
+
+import yamlutils "github.com/kyverno/kyverno/pkg/utils/yaml"
+
+func legacyLoader(_ string, content []byte) (*LoaderResults, error) {
+	policies, vaps, bindings, err := yamlutils.GetPolicy(content)
+	if err != nil {
+		return nil, err
+	}
+
+	return &LoaderResults{
+		Policies:    policies,
+		VAPs:        vaps,
+		VAPBindings: bindings,
+	}, nil
+}

cmd/cli/kubectl-kyverno/policy/load.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/go-git/go-billy/v5"
 	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
@@ -19,7 +20,7 @@ import (
 	resourceloader "github.com/kyverno/kyverno/ext/resource/loader"
 	extyaml "github.com/kyverno/kyverno/ext/yaml"
 	"github.com/kyverno/kyverno/pkg/utils/git"
-	yamlutils "github.com/kyverno/kyverno/pkg/utils/yaml"
+	"github.com/pkg/errors"
 	"k8s.io/api/admissionregistration/v1alpha1"
 	"k8s.io/api/admissionregistration/v1beta1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -37,198 +38,210 @@ var (
 	clusterPolicyV2       = schema.GroupVersion(kyvernov2beta1.GroupVersion).WithKind("ClusterPolicy")
 	vapV1alpha1           = v1alpha1.SchemeGroupVersion.WithKind("ValidatingAdmissionPolicy")
 	vapV1Beta1            = v1beta1.SchemeGroupVersion.WithKind("ValidatingAdmissionPolicy")
-	vapBidningV1alpha1    = v1alpha1.SchemeGroupVersion.WithKind("ValidatingAdmissionPolicyBinding")
+	vapBindingV1alpha1    = v1alpha1.SchemeGroupVersion.WithKind("ValidatingAdmissionPolicyBinding")
-	vapBidningV1beta1     = v1beta1.SchemeGroupVersion.WithKind("ValidatingAdmissionPolicyBinding")
+	vapBindingV1beta1     = v1beta1.SchemeGroupVersion.WithKind("ValidatingAdmissionPolicyBinding")
-	LegacyLoader          = yamlutils.GetPolicy
+	LegacyLoader          = legacyLoader
 	KubectlValidateLoader = kubectlValidateLoader
-	defaultLoader         = func(bytes []byte) ([]kyvernov1.PolicyInterface, []v1alpha1.ValidatingAdmissionPolicy, []v1alpha1.ValidatingAdmissionPolicyBinding, error) {
+	defaultLoader         = func(path string, bytes []byte) (*LoaderResults, error) {
 		if experimental.UseKubectlValidate() {
-			return KubectlValidateLoader(bytes)
+			return KubectlValidateLoader(path, bytes)
 		} else {
-			return LegacyLoader(bytes)
+			return LegacyLoader(path, bytes)
 		}
 	}
 )
 
-type loader = func([]byte) ([]kyvernov1.PolicyInterface, []v1alpha1.ValidatingAdmissionPolicy, []v1alpha1.ValidatingAdmissionPolicyBinding, error)
+type LoaderError struct {
+	Path  string
+	Error error
+}
 
-func Load(fs billy.Filesystem, resourcePath string, paths ...string) ([]kyvernov1.PolicyInterface, []v1alpha1.ValidatingAdmissionPolicy, []v1alpha1.ValidatingAdmissionPolicyBinding, error) {
+type LoaderResults struct {
+	Policies       []kyvernov1.PolicyInterface
+	VAPs           []v1alpha1.ValidatingAdmissionPolicy
+	VAPBindings    []v1alpha1.ValidatingAdmissionPolicyBinding
+	NonFatalErrors []LoaderError
+}
+
+func (l *LoaderResults) merge(results *LoaderResults) {
+	if results == nil {
+		return
+	}
+	l.Policies = append(l.Policies, results.Policies...)
+	l.VAPs = append(l.VAPs, results.VAPs...)
+	l.VAPBindings = append(l.VAPBindings, results.VAPBindings...)
+	l.NonFatalErrors = append(l.NonFatalErrors, results.NonFatalErrors...)
+}
+
+func (l *LoaderResults) addError(path string, err error) {
+	l.NonFatalErrors = append(l.NonFatalErrors, LoaderError{
+		Path:  path,
+		Error: err,
+	})
+}
+
+type loader = func(string, []byte) (*LoaderResults, error)
+
+func Load(fs billy.Filesystem, resourcePath string, paths ...string) (*LoaderResults, error) {
 	return LoadWithLoader(nil, fs, resourcePath, paths...)
 }
 
-func LoadWithLoader(loader loader, fs billy.Filesystem, resourcePath string, paths ...string) ([]kyvernov1.PolicyInterface, []v1alpha1.ValidatingAdmissionPolicy, []v1alpha1.ValidatingAdmissionPolicyBinding, error) {
+func LoadWithLoader(loader loader, fs billy.Filesystem, resourcePath string, paths ...string) (*LoaderResults, error) {
 	if loader == nil {
 		loader = defaultLoader
 	}
-	var pols []kyvernov1.PolicyInterface
+
-	var vaps []v1alpha1.ValidatingAdmissionPolicy
+	aggregateResults := &LoaderResults{}
-	var vapBindings []v1alpha1.ValidatingAdmissionPolicyBinding
 	for _, path := range paths {
+		var err error
+		var results *LoaderResults
 		if source.IsStdin(path) {
-			p, v, b, err := stdinLoad(loader)
+			results, err = stdinLoad(loader)
-			if err != nil {
-				return nil, nil, nil, err
-			}
-			pols = append(pols, p...)
-			vaps = append(vaps, v...)
-			vapBindings = append(vapBindings, b...)
 		} else if fs != nil {
-			p, v, b, err := gitLoad(loader, fs, filepath.Join(resourcePath, path))
+			results, err = gitLoad(loader, fs, filepath.Join(resourcePath, path))
-			if err != nil {
-				return nil, nil, nil, err
-			}
-			pols = append(pols, p...)
-			vaps = append(vaps, v...)
-			vapBindings = append(vapBindings, b...)
 		} else if source.IsHttp(path) {
-			p, v, b, err := httpLoad(loader, path)
+			results, err = httpLoad(loader, path)
-			if err != nil {
-				return nil, nil, nil, err
-			}
-			pols = append(pols, p...)
-			vaps = append(vaps, v...)
-			vapBindings = append(vapBindings, b...)
 		} else {
-			p, v, b, err := fsLoad(loader, path)
+			results, err = fsLoad(loader, path)
-			if err != nil {
-				return nil, nil, nil, err
-			}
-			pols = append(pols, p...)
-			vaps = append(vaps, v...)
-			vapBindings = append(vapBindings, b...)
 		}
+		if err != nil {
+			return nil, err
+		}
+		aggregateResults.merge(results)
 	}
 
 	// It's hard to use apply with the fake client, so disable all server side
 	// https://github.com/kubernetes/kubernetes/issues/99953
-	for _, policy := range pols {
+	for _, policy := range aggregateResults.Policies {
 		policy.GetSpec().UseServerSideApply = false
 	}
 
-	return pols, vaps, vapBindings, nil
+	return aggregateResults, nil
 }
 
-func kubectlValidateLoader(content []byte) ([]kyvernov1.PolicyInterface, []v1alpha1.ValidatingAdmissionPolicy, []v1alpha1.ValidatingAdmissionPolicyBinding, error) {
+func kubectlValidateLoader(path string, content []byte) (*LoaderResults, error) {
 	documents, err := extyaml.SplitDocuments(content)
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, err
 	}
-	var policies []kyvernov1.PolicyInterface
+	results := &LoaderResults{}
-	var vaps []v1alpha1.ValidatingAdmissionPolicy
-	var vapBindings []v1alpha1.ValidatingAdmissionPolicyBinding
 	for _, document := range documents {
 		gvk, untyped, err := factory.Load(document)
 		if err != nil {
-			return nil, nil, nil, err
+			msg := err.Error()
+			if strings.Contains(msg, "Invalid value: value provided for unknown field") {
+				return nil, err
+			}
+			// skip non-Kubernetes YAMLs and invalid types
+			results.addError(path, err)
+			continue
 		}
 		switch gvk {
 		case policyV1, policyV2:
 			typed, err := convert.To[kyvernov1.Policy](untyped)
 			if err != nil {
-				return nil, nil, nil, err
+				return nil, err
 			}
-			policies = append(policies, typed)
+			results.Policies = append(results.Policies, typed)
 		case clusterPolicyV1, clusterPolicyV2:
 			typed, err := convert.To[kyvernov1.ClusterPolicy](untyped)
 			if err != nil {
-				return nil, nil, nil, err
+				return nil, err
 			}
-			policies = append(policies, typed)
+			results.Policies = append(results.Policies, typed)
 		case vapV1alpha1, vapV1Beta1:
 			typed, err := convert.To[v1alpha1.ValidatingAdmissionPolicy](untyped)
 			if err != nil {
-				return nil, nil, nil, err
+				return nil, err
 			}
-			vaps = append(vaps, *typed)
+			results.VAPs = append(results.VAPs, *typed)
-		case vapBidningV1alpha1, vapBidningV1beta1:
+		case vapBindingV1alpha1, vapBindingV1beta1:
 			typed, err := convert.To[v1alpha1.ValidatingAdmissionPolicyBinding](untyped)
 			if err != nil {
-				return nil, nil, nil, err
+				return nil, err
 			}
-			vapBindings = append(vapBindings, *typed)
+			results.VAPBindings = append(results.VAPBindings, *typed)
 		default:
-			return nil, nil, nil, fmt.Errorf("policy type not supported %s", gvk)
+			return nil, fmt.Errorf("policy type not supported %s", gvk)
 		}
 	}
-	return policies, vaps, vapBindings, nil
+	return results, nil
 }
 
-func fsLoad(loader loader, path string) ([]kyvernov1.PolicyInterface, []v1alpha1.ValidatingAdmissionPolicy, []v1alpha1.ValidatingAdmissionPolicyBinding, error) {
+func fsLoad(loader loader, path string) (*LoaderResults, error) {
-	var pols []kyvernov1.PolicyInterface
-	var vaps []v1alpha1.ValidatingAdmissionPolicy
-	var vapBindings []v1alpha1.ValidatingAdmissionPolicyBinding
 	fi, err := os.Stat(filepath.Clean(path))
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, err
 	}
+	if strings.HasPrefix(fi.Name(), ".") {
+		// skip hidden files and dirs
+		return nil, err
+	}
+	aggregateResults := &LoaderResults{}
 	if fi.IsDir() {
 		files, err := os.ReadDir(path)
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, errors.Wrapf(err, "failed to read %s", path)
 		}
 		for _, file := range files {
-			p, v, b, err := fsLoad(loader, filepath.Join(path, file.Name()))
+			results, err := fsLoad(loader, filepath.Join(path, file.Name()))
 			if err != nil {
-				return nil, nil, nil, err
+				return nil, errors.Wrapf(err, "failed to load %s", path)
 			}
-			pols = append(pols, p...)
+			aggregateResults.merge(results)
-			vaps = append(vaps, v...)
-			vapBindings = append(vapBindings, b...)
 		}
 	} else if git.IsYaml(fi) {
-		fileBytes, err := os.ReadFile(filepath.Clean(path)) // #nosec G304
+		fileBytes, err := os.ReadFile(filepath.Clean(path))
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, errors.Wrapf(err, "failed to read file %s", path)
 		}
-		p, v, b, err := loader(fileBytes)
+		results, err := loader(path, fileBytes)
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, errors.Wrapf(err, "failed to load file %s", path)
 		}
-		pols = append(pols, p...)
+		aggregateResults.merge(results)
-		vaps = append(vaps, v...)
-		vapBindings = append(vapBindings, b...)
 	}
-	return pols, vaps, vapBindings, nil
+	return aggregateResults, nil
 }
 
-func httpLoad(loader loader, path string) ([]kyvernov1.PolicyInterface, []v1alpha1.ValidatingAdmissionPolicy, []v1alpha1.ValidatingAdmissionPolicyBinding, error) {
+func httpLoad(loader loader, path string) (*LoaderResults, error) {
 	// We accept here that a random URL might be called based on user provided input.
 	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, path, nil)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to process %v: %v", path, err)
+		return nil, fmt.Errorf("failed to process %v: %v", path, err)
 	}
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to process %v: %v", path, err)
+		return nil, fmt.Errorf("failed to process %v: %v", path, err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return nil, nil, nil, fmt.Errorf("failed to process %v: %v", path, err)
+		return nil, fmt.Errorf("failed to process %v: %v", path, err)
 	}
 	fileBytes, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to process %v: %v", path, err)
+		return nil, fmt.Errorf("failed to process %v: %v", path, err)
 	}
-	return loader(fileBytes)
+	return loader(path, fileBytes)
 }
 
-func gitLoad(loader loader, fs billy.Filesystem, path string) ([]kyvernov1.PolicyInterface, []v1alpha1.ValidatingAdmissionPolicy, []v1alpha1.ValidatingAdmissionPolicyBinding, error) {
+func gitLoad(loader loader, fs billy.Filesystem, path string) (*LoaderResults, error) {
 	file, err := fs.Open(path)
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, err
 	}
 	fileBytes, err := io.ReadAll(file)
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, err
 	}
-	return loader(fileBytes)
+	return loader(path, fileBytes)
 }
 
-func stdinLoad(loader loader) ([]kyvernov1.PolicyInterface, []v1alpha1.ValidatingAdmissionPolicy, []v1alpha1.ValidatingAdmissionPolicyBinding, error) {
+func stdinLoad(loader loader) (*LoaderResults, error) {
 	policyStr := ""
 	scanner := bufio.NewScanner(os.Stdin)
 	for scanner.Scan() {
 		policyStr = policyStr + scanner.Text() + "\n"
 	}
-	return loader([]byte(policyStr))
+	return loader("-", []byte(policyStr))
 }

cmd/cli/kubectl-kyverno/policy/load_test.go

@@ -31,7 +31,7 @@ func TestLoad(t *testing.T) {
 	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, _, _, err := Load(tt.fs, tt.resourcePath, tt.paths...)
+			_, err := Load(tt.fs, tt.resourcePath, tt.paths...)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Load() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -40,6 +40,44 @@ func TestLoad(t *testing.T) {
 	}
 }
 
+func TestLoadInvalid(t *testing.T) {
+	tests := []struct {
+		name         string
+		fs           billy.Filesystem
+		resourcePath string
+		paths        []string
+		wantErr      bool
+		count        int
+	}{{
+		name:         "invalid policy resources",
+		fs:           nil,
+		resourcePath: "",
+		paths:        []string{"../_testdata/policies-invalid/"},
+		wantErr:      false,
+		count:        0,
+	}, {
+		name:         "mixed policy resources",
+		fs:           nil,
+		resourcePath: "",
+		paths:        []string{"../_testdata/policies-mixed/"},
+		wantErr:      false,
+		count:        2,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			results, err := Load(tt.fs, tt.resourcePath, tt.paths...)
+			if tt.wantErr {
+				assert.NotNil(t, err, "result mismatch")
+			} else {
+				assert.NotNil(t, results)
+				if results != nil {
+					assert.Equal(t, tt.count, len(results.Policies), "policy count mismatch")
+				}
+			}
+		})
+	}
+}
+
 func TestLoadWithKubectlValidate(t *testing.T) {
 	tests := []struct {
 		name         string
@@ -87,13 +125,13 @@ func TestLoadWithKubectlValidate(t *testing.T) {
 	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			policies, vaps, _, err := LoadWithLoader(KubectlValidateLoader, tt.fs, tt.resourcePath, tt.paths...)
+			results, err := LoadWithLoader(KubectlValidateLoader, tt.fs, tt.resourcePath, tt.paths...)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Load() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			if tt.checks != nil {
-				tt.checks(t, policies, vaps)
+				tt.checks(t, results.Policies, results.VAPs)
 			}
 		})
 	}

cmd/cli/kubectl-kyverno/policy/variables_test.go

@@ -11,10 +11,10 @@ import (
 func TestExtractVariables(t *testing.T) {
 	loadPolicy := func(path string) kyvernov1.PolicyInterface {
 		t.Helper()
-		policies, _, _, err := Load(nil, "", path)
+		results, err := Load(nil, "", path)
 		assert.NoError(t, err)
-		assert.Equal(t, len(policies), 1)
+		assert.Equal(t, len(results.Policies), 1)
-		return policies[0]
+		return results.Policies[0]
 
 	}
 	tests := []struct {

cmd/cli/kubectl-kyverno/report/report_test.go

@@ -14,10 +14,10 @@ import (
 )
 
 func TestComputeClusterPolicyReports(t *testing.T) {
-	policies, _, _, err := policy.Load(nil, "", "../_testdata/policies/cpol-pod-requirements.yaml")
+	results, err := policy.Load(nil, "", "../_testdata/policies/cpol-pod-requirements.yaml")
 	assert.NilError(t, err)
-	assert.Equal(t, len(policies), 1)
+	assert.Equal(t, len(results.Policies), 1)
-	policy := policies[0]
+	policy := results.Policies[0]
 	er := engineapi.EngineResponse{}
 	er = er.WithPolicy(engineapi.NewKyvernoPolicy(policy))
 	er.PolicyResponse.Add(
@@ -48,10 +48,10 @@ func TestComputeClusterPolicyReports(t *testing.T) {
 }
 
 func TestComputePolicyReports(t *testing.T) {
-	policies, _, _, err := policy.Load(nil, "", "../_testdata/policies/pol-pod-requirements.yaml")
+	results, err := policy.Load(nil, "", "../_testdata/policies/pol-pod-requirements.yaml")
 	assert.NilError(t, err)
-	assert.Equal(t, len(policies), 1)
+	assert.Equal(t, len(results.Policies), 1)
-	policy := policies[0]
+	policy := results.Policies[0]
 	er := engineapi.EngineResponse{}
 	er = er.WithPolicy(engineapi.NewKyvernoPolicy(policy))
 	er.PolicyResponse.Add(
@@ -83,10 +83,10 @@ func TestComputePolicyReports(t *testing.T) {
 }
 
 func TestComputePolicyReportResultsPerPolicyOld(t *testing.T) {
-	policies, _, _, err := policy.Load(nil, "", "../_testdata/policies/cpol-pod-requirements.yaml")
+	loaderResults, err := policy.Load(nil, "", "../_testdata/policies/cpol-pod-requirements.yaml")
 	assert.NilError(t, err)
-	assert.Equal(t, len(policies), 1)
+	assert.Equal(t, len(loaderResults.Policies), 1)
-	policy := policies[0]
+	policy := loaderResults.Policies[0]
 	er := engineapi.EngineResponse{}
 	er = er.WithPolicy(engineapi.NewKyvernoPolicy(policy))
 	er.PolicyResponse.Add(
@@ -161,10 +161,10 @@ func TestMergeClusterReport(t *testing.T) {
 }
 
 func TestComputePolicyReportResult(t *testing.T) {
-	policies, _, _, err := policy.Load(nil, "", "../_testdata/policies/cpol-pod-requirements.yaml")
+	results, err := policy.Load(nil, "", "../_testdata/policies/cpol-pod-requirements.yaml")
 	assert.NilError(t, err)
-	assert.Equal(t, len(policies), 1)
+	assert.Equal(t, len(results.Policies), 1)
-	policy := policies[0]
+	policy := results.Policies[0]
 	tests := []struct {
 		name           string
 		auditWarn      bool
@@ -280,10 +280,10 @@ func TestComputePolicyReportResult(t *testing.T) {
 }
 
 func TestPSSComputePolicyReportResult(t *testing.T) {
-	policies, _, _, err := policy.Load(nil, "", "../_testdata/policies/restricted.yaml")
+	results, err := policy.Load(nil, "", "../_testdata/policies/restricted.yaml")
 	assert.NilError(t, err)
-	assert.Equal(t, len(policies), 1)
+	assert.Equal(t, len(results.Policies), 1)
-	policy := policies[0]
+	policy := results.Policies[0]
 	tests := []struct {
 		name           string
 		auditWarn      bool