refactor: add cli fix package (#8400)

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

cmd/cli/kubectl-kyverno/commands/fix/test/options.go

@@ -6,8 +6,9 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"reflect"
 
-	testapi "github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/apis/test"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/fix"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test"
 	"sigs.k8s.io/yaml"
 )
@@ -45,66 +46,24 @@ func (o options) execute(out io.Writer, dirs ...string) error {
 			fmt.Fprintln(out)
 			continue
 		}
-		test := testCase.Test
-		needsSave := false
-		if test.Name == "" {
+		fixed := *testCase.Test
+		if fixed.Name == "" {
 			fmt.Fprintln(out, "  WARNING: name is not set")
-			test.Name = filepath.Base(testCase.Path)
-			needsSave = true
+			fixed.Name = filepath.Base(testCase.Path)
 		}
-		if len(test.Policies) == 0 {
-			fmt.Fprintln(out, "  WARNING: test has no policies")
+		fixed, messages, err := fix.FixTest(fixed, o.compress)
+		for _, warning := range messages {
+			fmt.Fprintln(out, "  WARNING:", warning)
 		}
-		if len(test.Resources) == 0 {
-			fmt.Fprintln(out, "  WARNING: test has no resources")
-		}
-		for i := range test.Results {
-			result := &test.Results[i]
-			if result.Resource != "" && len(result.Resources) != 0 {
-				fmt.Fprintln(out, "  WARNING: test result should not use both `resource` and `resources` fields")
-			}
-			if result.Resource != "" {
-				fmt.Fprintln(out, "  WARNING: test result uses deprecated `resource` field, moving it into the `resources` field")
-				result.Resources = append(result.Resources, result.Resource)
-				result.Resource = ""
-				needsSave = true
-			}
-			if result.Namespace != "" {
-				fmt.Fprintln(out, "  WARNING: test result uses deprecated `namespace` field, replacing `policy` with a `<namespace>/<name>` pattern")
-				result.Policy = fmt.Sprintf("%s/%s", result.Namespace, result.Policy)
-				result.Namespace = ""
-				needsSave = true
-			}
-			if result.Status != "" && result.Result != "" {
-				fmt.Fprintln(out, "  ERROR: test result should not use both `status` and `result` fields")
-			}
-			if result.Status != "" && result.Result == "" {
-				fmt.Fprintln(out, "  WARNING: test result uses deprecated `status` field, moving it into the `result` field")
-				result.Result = result.Status
-				result.Status = ""
-				needsSave = true
-			}
-		}
-		if o.compress {
-			compressed := map[testapi.TestResultBase][]string{}
-			for _, result := range test.Results {
-				compressed[result.TestResultBase] = append(compressed[result.TestResultBase], result.Resources...)
-			}
-			if len(compressed) != len(test.Results) {
-				needsSave = true
-			}
-			test.Results = nil
-			for k, v := range compressed {
-				test.Results = append(test.Results, testapi.TestResult{
-					TestResultBase: k,
-					Resources:      v,
-				})
-			}
+		if err != nil {
+			fmt.Fprintln(out, "  ERROR:", err)
+			continue
 		}
+		needsSave := !reflect.DeepEqual(testCase.Test, &fixed)
 		if o.save && needsSave {
 			fmt.Fprintf(out, "  Saving test file (%s)...", testCase.Path)
 			fmt.Fprintln(out)
-			yamlBytes, err := yaml.Marshal(test)
+			yamlBytes, err := yaml.Marshal(fixed)
 			if err != nil {
 				fmt.Fprintf(out, "    ERROR: converting test to yaml: %s", err)
 				fmt.Fprintln(out)

cmd/cli/kubectl-kyverno/fix/test.go

@@ -0,0 +1,101 @@
+package fix
+
+import (
+	"errors"
+	"fmt"
+
+	testapi "github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/apis/test"
+	"golang.org/x/exp/slices"
+)
+
+func FixTest(test testapi.Test, compress bool) (testapi.Test, []string, error) {
+	var messages []string
+	if test.Name == "" {
+		messages = append(messages, "name is not set")
+	}
+	if len(test.Policies) == 0 {
+		messages = append(messages, "test has no policies")
+	}
+	if len(test.Resources) == 0 {
+		messages = append(messages, "test has no resources")
+	}
+	var results []testapi.TestResult
+	for _, result := range test.Results {
+		if result.Resource != "" && len(result.Resources) != 0 {
+			messages = append(messages, "test result should not use both `resource` and `resources` fields")
+		}
+		if result.Resource != "" {
+			var resources []string
+			messages = append(messages, "test result uses deprecated `resource` field, moving it into the `resources` field")
+			resources = append(resources, result.Resources...)
+			resources = append(resources, result.Resource)
+			result.Resources = resources
+			result.Resource = ""
+		}
+		if result.Namespace != "" {
+			messages = append(messages, "test result uses deprecated `namespace` field, replacing `policy` with a `<namespace>/<name>` pattern")
+			result.Policy = fmt.Sprintf("%s/%s", result.Namespace, result.Policy)
+			result.Namespace = ""
+		}
+		if result.Status != "" && result.Result != "" {
+			return test, messages, errors.New("test result should not use both `status` and `result` fields")
+		}
+		if result.Status != "" && result.Result == "" {
+			messages = append(messages, "test result uses deprecated `status` field, moving it into the `result` field")
+			result.Result = result.Status
+			result.Status = ""
+		}
+		results = append(results, result)
+	}
+	if compress {
+		compressed := map[testapi.TestResultBase][]string{}
+		for _, result := range results {
+			compressed[result.TestResultBase] = append(compressed[result.TestResultBase], result.Resources...)
+		}
+		results = nil
+		for k, v := range compressed {
+			results = append(results, testapi.TestResult{
+				TestResultBase: k,
+				Resources:      v,
+			})
+		}
+	}
+	slices.SortFunc(results, func(a, b testapi.TestResult) bool {
+		if a.Policy < b.Policy {
+			return true
+		}
+		if a.Rule < b.Rule {
+			return true
+		}
+		if a.Result < b.Result {
+			return true
+		}
+		if a.Kind < b.Kind {
+			return true
+		}
+		if a.PatchedResource < b.PatchedResource {
+			return true
+		}
+		if a.GeneratedResource < b.GeneratedResource {
+			return true
+		}
+		if a.CloneSourceResource < b.CloneSourceResource {
+			return true
+		}
+		slices.Sort(a.Resources)
+		slices.Sort(b.Resources)
+		if len(a.Resources) < len(b.Resources) {
+			return true
+		}
+		if len(a.Resources) == len(b.Resources) {
+			for i := range a.Resources {
+				if a.Resources[i] < b.Resources[i] {
+					return true
+				}
+			}
+		}
+		return false
+	})
+	test.Results = results
+	return test, messages, nil
+}

test/cli/registry/kyverno-test.yaml

@@ -4,6 +4,12 @@ policies:
 resources:
 - resources.yaml
 results:
+- kind: Pod
+  policy: check-image-base
+  resources:
+  - test-pod-with-trusted-registry
+  result: pass
+  rule: check-image-base-rule
 - kind: Pod
   policy: images
   resources:
@@ -11,9 +17,3 @@ results:
   - test-pod-with-trusted-registry
   result: pass
   rule: only-allow-trusted-images
-- kind: Pod
-  policy: check-image-base
-  resources:
-  - test-pod-with-trusted-registry
-  result: pass
-  rule: check-image-base-rule

test/cli/scenarios_to_cli/other/scenario_validate_healthChecks/kyverno-test.yaml

@@ -9,10 +9,10 @@ results:
   resources:
   - probe
   result: pass
-  rule: check-readinessProbe-exists
+  rule: check-livenessProbe-exists
 - kind: Pod
   policy: check-probe-exists
   resources:
   - probe
   result: pass
-  rule: check-livenessProbe-exists
+  rule: check-readinessProbe-exists

test/cli/test-generate/add-quota/kyverno-test.yaml

@@ -4,13 +4,6 @@ policies:
 resources:
 - resource.yaml
 results:
-- generatedResource: generatedResourceQuota.yaml
-  kind: Namespace
-  policy: add-ns-quota
-  resources:
-  - hello-world-namespace
-  result: pass
-  rule: generate-resourcequota
 - generatedResource: generatedLimitRange.yaml
   kind: Namespace
   policy: add-ns-quota
@@ -18,3 +11,10 @@ results:
   - hello-world-namespace
   result: pass
   rule: generate-limitrange
+- generatedResource: generatedResourceQuota.yaml
+  kind: Namespace
+  policy: add-ns-quota
+  resources:
+  - hello-world-namespace
+  result: pass
+  rule: generate-resourcequota

test/cli/test-generate/sync-multiple-resources/kyverno-test.yaml

@@ -1,22 +1,22 @@
 name: sync-controller-data
 policies:
-  - policy.yaml
+- policy.yaml
 resources:
-  - deployment.yaml
+- deployment.yaml
 results:
-  - policy: sync-controller-data
-    rule: sync-controller-secret
-    resources:
-      - kubernetes-cluster-controller
-    generatedResource: gen-secret.yaml
-    cloneSourceResource: secret.yaml
-    kind: Deployment
-    result: pass
-  - policy: sync-controller-data
-    rule: sync-controller-configmap
-    resources: 
-      - kubernetes-cluster-controller
-    generatedResource: gen-cm.yaml
-    cloneSourceResource: cm.yaml
-    kind: Deployment
-    result: pass
+- cloneSourceResource: cm.yaml
+  generatedResource: gen-cm.yaml
+  kind: Deployment
+  policy: sync-controller-data
+  resources:
+  - kubernetes-cluster-controller
+  result: pass
+  rule: sync-controller-configmap
+- cloneSourceResource: secret.yaml
+  generatedResource: gen-secret.yaml
+  kind: Deployment
+  policy: sync-controller-data
+  resources:
+  - kubernetes-cluster-controller
+  result: pass
+  rule: sync-controller-secret

test/cli/test-mutate/add-default-resources/kyverno-test.yaml

@@ -11,13 +11,6 @@ results:
   - nginx-demo1
   result: pass
   rule: add-default-requests
-- kind: Pod
-  patchedResource: patchedResource2.yaml
-  policy: add-default-resources
-  resources:
-  - nginx-demo2
-  result: skip
-  rule: add-default-requests
 - kind: Pod
   patchedResource: patchedResource3.yaml
   policy: add-default-resources
@@ -25,3 +18,10 @@ results:
   - nginx-demo3
   result: pass
   rule: add-default-requests
+- kind: Pod
+  patchedResource: patchedResource2.yaml
+  policy: add-default-resources
+  resources:
+  - nginx-demo2
+  result: skip
+  rule: add-default-requests

test/cli/test-mutate/connection-draining/kyverno-test.yaml

@@ -4,12 +4,6 @@ policies:
 resources:
 - resource.yaml
 results:
-- kind: Service
-  policy: disable-connection-draining
-  resources:
-  - nlb-aws-controller-no-attributes
-  result: skip
-  rule: clb
 - kind: Service
   patchedResource: patched.yaml
   policy: disable-connection-draining
@@ -17,3 +11,9 @@ results:
   - nlb-aws-controller-no-attributes
   result: pass
   rule: nlb-no-attributes
+- kind: Service
+  policy: disable-connection-draining
+  resources:
+  - nlb-aws-controller-no-attributes
+  result: skip
+  rule: clb

test/cli/test-mutate/foreach/kyverno-test.yaml

@@ -4,13 +4,6 @@ policies:
 resources:
 - resources.yaml
 results:
-- kind: Pod
-  patchedResource: patched-resource.yaml
-  policy: foreach-json-patch
-  resources:
-  - nginx
-  result: pass
-  rule: add-security-context
 - kind: Pod
   patchedResource: pod-updated-image.yaml
   policy: mutate-images
@@ -18,4 +11,11 @@ results:
   - mypod
   result: pass
   rule: test
+- kind: Pod
+  patchedResource: patched-resource.yaml
+  policy: foreach-json-patch
+  resources:
+  - nginx
+  result: pass
+  rule: add-security-context
 variables: values.yaml

test/cli/test-mutate/karpenter-annotations-to-nodeselector/kyverno-test.yaml

@@ -4,6 +4,12 @@ policies:
 resources:
 - resource.yaml
 results:
+- kind: Pod
+  policy: karpenter-annotations-to-nodeselector
+  resources:
+  - soft-pod-antiaffinity-1-copy
+  result: pass
+  rule: hard-nodeselector-lifecycle-on-demand
 - kind: Pod
   patchedResource: patched.yaml
   policy: karpenter-annotations-to-nodeselector
@@ -11,9 +17,3 @@ results:
   - soft-pod-antiaffinity-1
   result: pass
   rule: hard-nodeselector-lifecycle-on-demand
-- kind: Pod
-  policy: karpenter-annotations-to-nodeselector
-  resources:
-  - soft-pod-antiaffinity-1-copy
-  result: pass
-  rule: hard-nodeselector-lifecycle-on-demand

test/cli/test-mutate/kyverno-test.yaml

@@ -5,12 +5,26 @@ resources:
 - resource.yaml
 results:
 - kind: Pod
-  patchedResource: patchedResource3.yaml
+  patchedResource: patchedResource2.yaml
   policy: add-label
   resources:
-  - production/same-name-but-diff-namespace
+  - testing/same-name-but-diff-namespace
   result: pass
   rule: add-label
+- kind: Pod
+  patchedResource: patchedResource1.yaml
+  policy: add-label
+  resources:
+  - practice/resource-equal-to-patch-res-for-cp
+  result: skip
+  rule: add-label
+- kind: Pod
+  patchedResource: patched-resource.yaml
+  policy: example
+  resources:
+  - example
+  result: pass
+  rule: object_from_lists
 - kind: Deployment
   patchedResource: patchedResource4.yaml
   policy: add-label
@@ -18,6 +32,13 @@ results:
   - mydeploy
   result: pass
   rule: add-label
+- kind: Pod
+  patchedResource: patchedResource3.yaml
+  policy: add-label
+  resources:
+  - production/same-name-but-diff-namespace
+  result: pass
+  rule: add-label
 - kind: Pod
   patchedResource: patchedResource6.yaml
   policy: add-label
@@ -32,24 +53,3 @@ results:
   - same-name-but-diff-namespace
   result: pass
   rule: add-ndots
-- kind: Pod
-  patchedResource: patched-resource.yaml
-  policy: example
-  resources:
-  - example
-  result: pass
-  rule: object_from_lists
-- kind: Pod
-  patchedResource: patchedResource1.yaml
-  policy: add-label
-  resources:
-  - practice/resource-equal-to-patch-res-for-cp
-  result: skip
-  rule: add-label
-- kind: Pod
-  patchedResource: patchedResource2.yaml
-  policy: add-label
-  resources:
-  - testing/same-name-but-diff-namespace
-  result: pass
-  rule: add-label

test/cli/test/anypattern_skip_error/kyverno-test.yaml

@@ -7,9 +7,8 @@ results:
 - kind: Service
   policy: validate-service-loadbalancer
   resources:
-  - service-public-pass
-  - service-public-2-pass
-  result: pass
+  - service-clusterip-skip
+  result: skip
   rule: check-loadbalancer-public
 - kind: Service
   policy: validate-service-loadbalancer
@@ -20,6 +19,7 @@ results:
 - kind: Service
   policy: validate-service-loadbalancer
   resources:
-  - service-clusterip-skip
-  result: skip
+  - service-public-2-pass
+  - service-public-pass
+  result: pass
   rule: check-loadbalancer-public

test/cli/test/autogen-values/kyverno-test.yaml

@@ -1,19 +1,19 @@
 name: test
 policies:
-  - policy.yaml
-variables: values.yaml
+- policy.yaml
 resources:
-  - resource.yaml
+- resource.yaml
 results:
-  - policy: test-policy
-    rule: test-rule
-    resources:
-    - pod
-    kind: Pod
-    result: pass
-  - policy: test-policy
-    rule: test-rule
-    resources:
-    - deployment
-    kind: Deployment
-    result: pass
+- kind: Deployment
+  policy: test-policy
+  resources:
+  - deployment
+  result: pass
+  rule: test-rule
+- kind: Pod
+  policy: test-policy
+  resources:
+  - pod
+  result: pass
+  rule: test-rule
+variables: values.yaml

test/cli/test/autogen/kyverno-test.yaml

@@ -4,51 +4,51 @@ policies:
 resources:
 - resources.yaml
 results:
-- kind: Pod
-  policy: require-common-labels
-  resources:
-  - pod-with-labels
-  result: pass
-  rule: check-for-labels
-- kind: Pod
-  policy: require-common-labels
-  resources:
-  - pod-missing-labels
-  result: fail
-  rule: check-for-labels
-- kind: Deployment
-  policy: require-common-labels
-  resources:
-  - deployment-with-labels
-  result: pass
-  rule: check-for-labels
-- kind: Deployment
-  policy: require-common-labels
-  resources:
-  - deployment-missing-labels
-  result: fail
-  rule: check-for-labels
 - kind: StatefulSet
   policy: require-common-labels
   resources:
   - StatefulSet-with-labels
   result: pass
   rule: check-for-labels
-- kind: StatefulSet
-  policy: require-common-labels
-  resources:
-  - StatefulSet-without-labels
-  result: fail
-  rule: check-for-labels
-- kind: CronJob
-  policy: require-common-labels
-  resources:
-  - cronjob-with-labels
-  result: pass
-  rule: check-for-labels
 - kind: CronJob
   policy: require-common-labels
   resources:
   - cronjob-without-labels
   result: fail
   rule: check-for-labels
+- kind: Deployment
+  policy: require-common-labels
+  resources:
+  - deployment-missing-labels
+  result: fail
+  rule: check-for-labels
+- kind: Pod
+  policy: require-common-labels
+  resources:
+  - pod-missing-labels
+  result: fail
+  rule: check-for-labels
+- kind: CronJob
+  policy: require-common-labels
+  resources:
+  - cronjob-with-labels
+  result: pass
+  rule: check-for-labels
+- kind: Deployment
+  policy: require-common-labels
+  resources:
+  - deployment-with-labels
+  result: pass
+  rule: check-for-labels
+- kind: Pod
+  policy: require-common-labels
+  resources:
+  - pod-with-labels
+  result: pass
+  rule: check-for-labels
+- kind: StatefulSet
+  policy: require-common-labels
+  resources:
+  - StatefulSet-without-labels
+  result: fail
+  rule: check-for-labels

test/cli/test/context-entries/kyverno-test.yaml

@@ -4,12 +4,6 @@ policies:
 resources:
 - resources.yaml
 results:
-- kind: Pod
-  policy: example
-  resources:
-  - example
-  result: pass
-  rule: defined-value
 - kind: Pod
   policy: example
   resources:
@@ -27,13 +21,13 @@ results:
   resources:
   - example
   result: pass
-  rule: defined-value-with-variable
+  rule: defined-jmespath-with-default-variable
 - kind: Pod
   policy: example
   resources:
   - example
   result: pass
-  rule: defined-jmespath-with-default-variable
+  rule: defined-value
 - kind: Pod
   policy: example
   resources:
@@ -51,13 +45,7 @@ results:
   resources:
   - example
   result: pass
-  rule: value-override
-- kind: Pod
-  policy: example
-  resources:
-  - example
-  result: pass
-  rule: wildcard-match
+  rule: defined-value-with-variable
 - kind: Pod
   policy: example
   resources:
@@ -70,3 +58,15 @@ results:
   - example
   result: pass
   rule: unused-var
+- kind: Pod
+  policy: example
+  resources:
+  - example
+  result: pass
+  rule: value-override
+- kind: Pod
+  policy: example
+  resources:
+  - example
+  result: pass
+  rule: wildcard-match

test/cli/test/context-foreach/kyverno-test.yaml

@@ -7,13 +7,13 @@ results:
 - kind: Pod
   policy: block-images
   resources:
-  - good-pod
-  result: pass
+  - bad-pod
+  result: fail
   rule: block-images
 - kind: Pod
   policy: block-images
   resources:
-  - bad-pod
-  result: fail
+  - good-pod
+  result: pass
   rule: block-images
 variables: values.yaml

test/cli/test/custom-functions/kyverno-test.yaml

@@ -4,54 +4,30 @@ policies:
 resources:
 - resources.yaml
 results:
-- kind: Secret
-  policy: base64
-  resources:
-  - base64-test-match
-  result: pass
-  rule: secret-value-must-match-label
-- kind: Secret
-  policy: base64
-  resources:
-  - base64-test-no-match
-  result: fail
-  rule: secret-value-must-match-label
-- kind: Namespace
-  policy: pattern-match
-  resources:
-  - pattern-match-test-match
-  result: pass
-  rule: label-must-match-pattern
-- kind: Namespace
-  policy: pattern-match
-  resources:
-  - pattern-match-test-no-match
-  result: fail
-  rule: label-must-match-pattern
 - kind: Pod
   policy: path-canonicalize
   resources:
   - mount-containerd-sock
   result: fail
   rule: disallow-mount-containerd-sock
-- kind: ConfigMap
-  policy: test-parse-json
+- kind: Secret
+  policy: base64
   resources:
-  - valid-test
+  - base64-test-no-match
+  result: fail
+  rule: secret-value-must-match-label
+- kind: Secret
+  policy: base64
+  resources:
+  - base64-test-match
   result: pass
-  rule: test-json-parsing-jmespath
+  rule: secret-value-must-match-label
 - kind: ConfigMap
   policy: test-parse-json
   resources:
   - invalid-test
   result: fail
   rule: test-json-parsing-jmespath
-- kind: ConfigMap
-  policy: test-parse-yaml
-  resources:
-  - valid-yaml-test
-  result: pass
-  rule: test-yaml-parsing-jmespath
 - kind: ConfigMap
   policy: test-parse-yaml
   resources:
@@ -59,7 +35,13 @@ results:
   result: fail
   rule: test-yaml-parsing-jmespath
 - kind: ConfigMap
-  policy: test-parse-yaml-array
+  policy: test-parse-json
+  resources:
+  - valid-test
+  result: pass
+  rule: test-json-parsing-jmespath
+- kind: ConfigMap
+  policy: test-parse-yaml
   resources:
   - valid-yaml-test
   result: pass
@@ -76,3 +58,21 @@ results:
   - test-x509-configmap
   result: fail
   rule: test-x509-decode
+- kind: ConfigMap
+  policy: test-parse-yaml-array
+  resources:
+  - valid-yaml-test
+  result: pass
+  rule: test-yaml-parsing-jmespath
+- kind: Namespace
+  policy: pattern-match
+  resources:
+  - pattern-match-test-no-match
+  result: fail
+  rule: label-must-match-pattern
+- kind: Namespace
+  policy: pattern-match
+  resources:
+  - pattern-match-test-match
+  result: pass
+  rule: label-must-match-pattern

test/cli/test/deny-modify-platform-label-2/kyverno-test.yaml

@@ -13,8 +13,8 @@ results:
 - kind: Role
   policy: deny-modify-platform-label
   resources:
-  - my-role-without-platform
   - my-role-with-platform-false
+  - my-role-without-platform
   result: skip
   rule: deny-modify-platform-role
 variables: variables.yaml

test/cli/test/disallow-service/kyverno-test.yaml

@@ -1,16 +1,18 @@
 name: disallow-service
 policies:
-  - policy.yaml
+- policy.yaml
 resources:
-  - resource.yaml
+- resource.yaml
 results:
-  - policy: disallow-service
-    rule: disallow-service
-    resources: [svc1]
-    kind: Service
-    result: skip
-  - policy: disallow-service
-    rule: disallow-service
-    resources: [svc2]
-    kind: Service
-    result: fail
+- kind: Service
+  policy: disallow-service
+  resources:
+  - svc2
+  result: fail
+  rule: disallow-service
+- kind: Service
+  policy: disallow-service
+  resources:
+  - svc1
+  result: skip
+  rule: disallow-service

test/cli/test/foreach/kyverno-test.yaml

@@ -5,17 +5,23 @@ resources:
 - resources.yaml
 results:
 - kind: Pod
-  policy: validate-empty-dir-mountpath
+  policy: validate-image-list
   resources:
   - test-pod
-  - test-pod2
-  result: pass
-  rule: check-mount-paths
+  - test-pod-ghcr
+  result: fail
+  rule: check-image
+- kind: Pod
+  policy: validate-empty-dir-resources
+  resources:
+  - test-pod-with-gke-vol
+  result: skip
+  rule: check-resources
 - kind: Pod
   policy: validate-empty-dir-resources
   resources:
-  - test-pod-bad-mount
   - test-pod
+  - test-pod-bad-mount
   result: fail
   rule: check-resources
 - kind: Pod
@@ -26,18 +32,12 @@ results:
   result: pass
   rule: check-resources
 - kind: Pod
-  policy: validate-empty-dir-resources
-  resources:
-  - test-pod-with-gke-vol
-  result: skip
-  rule: check-resources
-- kind: Pod
-  policy: validate-image-list
+  policy: validate-empty-dir-mountpath
   resources:
   - test-pod
-  - test-pod-ghcr
-  result: fail
-  rule: check-image
+  - test-pod2
+  result: pass
+  rule: check-mount-paths
 - kind: Pod
   policy: validate-image-list-error
   resources:

test/cli/test/images/signatures/kyverno-test.yaml

@@ -7,12 +7,12 @@ results:
 - kind: Pod
   policy: verify-signature
   resources:
-  - signed
-  result: pass
+  - unsigned
+  result: fail
   rule: check-static-key
 - kind: Pod
   policy: verify-signature
   resources:
-  - unsigned
-  result: fail
+  - signed
+  result: pass
   rule: check-static-key

test/cli/test/images/verify-signature/kyverno-test.yaml

@@ -10,10 +10,10 @@ results:
   - signed
   result: pass
   rule: verify-signature
-- kind: Pod
-  policy: check-image
+- kind: DataVolume
+  policy: check-data-volume-image
   resources:
-  - unsigned
+  - unsigned-registry-image-datavolume
   result: fail
   rule: verify-signature
 - kind: DataVolume
@@ -22,9 +22,9 @@ results:
   - signed-registry-image-datavolume
   result: pass
   rule: verify-signature
-- kind: DataVolume
-  policy: check-data-volume-image
+- kind: Pod
+  policy: check-image
   resources:
-  - unsigned-registry-image-datavolume
+  - unsigned
   result: fail
   rule: verify-signature

test/cli/test/jmespath-brackets/kyverno-test.yaml

@@ -4,20 +4,6 @@ policies:
 resources:
 - resources.yaml
 results:
-- kind: Pod
-  policy: default/test-jmespath
-  resources:
-  - test-valid1
-  - test-valid2
-  - test-valid3
-  result: pass
-  rule: test-jmespath
-- kind: Pod
-  policy: default/test-jmespath
-  resources:
-  - test-invalid
-  result: fail
-  rule: test-jmespath
 - kind: Namespace
   policy: namespace-validation
   resources:
@@ -30,3 +16,17 @@ results:
   - test-valid
   result: pass
   rule: namespace-validation
+- kind: Pod
+  policy: default/test-jmespath
+  resources:
+  - test-invalid
+  result: fail
+  rule: test-jmespath
+- kind: Pod
+  policy: default/test-jmespath
+  resources:
+  - test-valid1
+  - test-valid2
+  - test-valid3
+  result: pass
+  rule: test-jmespath

test/cli/test/limit-configmap-for-sa/kyverno-test.yaml

@@ -7,13 +7,13 @@ results:
 - kind: ConfigMap
   policy: limit-configmap-for-sa
   resources:
-  - any-namespace/any-configmap-name-good
-  result: fail
+  - any-namespace/any-configmap-name-bad
+  result: skip
   rule: limit-configmap-for-sa-developer
 - kind: ConfigMap
   policy: limit-configmap-for-sa
   resources:
-  - any-namespace/any-configmap-name-bad
-  result: skip
+  - any-namespace/any-configmap-name-good
+  result: fail
   rule: limit-configmap-for-sa-developer
 variables: variables.yaml

test/cli/test/mixed/kyverno-test.yaml

@@ -10,6 +10,12 @@ results:
   - user-foo/nodeselector-without-labels-on-mutation
   result: fail
   rule: ondemand-managed_by
+- kind: Pod
+  policy: ondemand
+  resources:
+  - user-space/nodeselector-with-labels-on-mutation
+  result: pass
+  rule: ondemand-managed_by
 - kind: Pod
   patchedResource: patched-resource.yaml
   policy: ondemand
@@ -17,9 +23,3 @@ results:
   - user-space/nodeselector-with-labels-on-mutation
   result: pass
   rule: ondemand-nodeselector
-- kind: Pod
-  policy: ondemand
-  resources:
-  - user-space/nodeselector-with-labels-on-mutation
-  result: pass
-  rule: ondemand-managed_by

test/cli/test/multiple_condition_keys/kyverno-test.yaml

@@ -7,12 +7,12 @@ results:
 - kind: Pod
   policy: test-multiple-key
   resources:
-  - test-resource-pass
-  result: pass
+  - test-resource-fail
+  result: fail
   rule: test-multiple-key
 - kind: Pod
   policy: test-multiple-key
   resources:
-  - test-resource-fail
-  result: fail
+  - test-resource-pass
+  result: pass
   rule: test-multiple-key

test/cli/test/mutate-keda-scaled-object/kyverno-test.yaml

@@ -1,23 +1,26 @@
 name: mutate-keda-scaled-object
 policies:
-  -  policy.yaml
+- policy.yaml
 resources:
-  -  resources.yaml
+- resources.yaml
 results:
-  - policy: keda-prometheus-serveraddress
-    rule: keda-prometheus-serveraddress
-    resources: [service-1]
-    patchedResource: patchedResource1.yaml
-    kind: ScaledObject
-    result: pass
-  - policy: keda-prometheus-serveraddress
-    rule: keda-prometheus-serveraddress
-    resources: [service-2]
-    patchedResource: patchedResource2.yaml
-    kind: ScaledObject
-    result: pass
-  - policy: keda-prometheus-serveraddress
-    rule: keda-prometheus-serveraddress
-    resources: [service-3]
-    kind: ScaledObject
-    result: skip
+- kind: ScaledObject
+  patchedResource: patchedResource1.yaml
+  policy: keda-prometheus-serveraddress
+  resources:
+  - service-1
+  result: pass
+  rule: keda-prometheus-serveraddress
+- kind: ScaledObject
+  policy: keda-prometheus-serveraddress
+  resources:
+  - service-3
+  result: skip
+  rule: keda-prometheus-serveraddress
+- kind: ScaledObject
+  patchedResource: patchedResource2.yaml
+  policy: keda-prometheus-serveraddress
+  resources:
+  - service-2
+  result: pass
+  rule: keda-prometheus-serveraddress

test/cli/test/nil-values-in-variables/exclude_namespaces_dynamically/kyverno-test.yaml

@@ -7,13 +7,13 @@ results:
 - kind: Pod
   policy: exclude-namespaces-example
   resources:
-  - bad-pod01
-  result: pass
+  - bad-pod02
+  result: error
   rule: exclude-namespaces-dynamically
 - kind: Pod
   policy: exclude-namespaces-example
   resources:
-  - bad-pod02
-  result: error
+  - bad-pod01
+  result: pass
   rule: exclude-namespaces-dynamically
 variables: values.yaml

test/cli/test/nil-values-in-variables/limit-duration/kyverno-test.yaml

@@ -7,12 +7,12 @@ results:
 - kind: Certificate
   policy: cert-manager-limit-duration
   resources:
-  - letsencrypt-crt
-  result: skip
+  - acme-crt
+  result: error
   rule: certificate-duration-max-100days
 - kind: Certificate
   policy: cert-manager-limit-duration
   resources:
-  - acme-crt
-  result: error
+  - letsencrypt-crt
+  result: skip
   rule: certificate-duration-max-100days

test/cli/test/policy-reports-skip-validation/kyverno-test.yaml

@@ -7,13 +7,13 @@ results:
 - kind: Pod
   policy: disallow-naked-pods
   resources:
-  - blank-skip
-  result: skip
+  - blank-fail
+  result: fail
   rule: validate-naked-pods
 - kind: Pod
   policy: disallow-naked-pods
   resources:
-  - blank-fail
-  result: fail
+  - blank-skip
+  result: skip
   rule: validate-naked-pods
 variables: values.yaml

test/cli/test/preconditions/kyverno-test.yaml

@@ -7,12 +7,12 @@ results:
 - kind: Pod
   policy: preconditions
   resources:
-  - test-valid
-  result: pass
+  - test-invalid
+  result: fail
   rule: any-rule
 - kind: Pod
   policy: preconditions
   resources:
-  - test-invalid
-  result: fail
+  - test-valid
+  result: pass
   rule: any-rule

test/cli/test/resource_lists/kyverno-test.yaml

@@ -1,19 +1,19 @@
 name: resource-lists
 policies:
-  -  policy.yaml
+- policy.yaml
 resources:
-  -  resource.yaml
+- resource.yaml
 results:
-  - policy: resource-lists
-    rule: require-image-tag
-    resources: 
-    - myapp-pod1
-    - myapp-pod2
-    kind: Pod
-    result: pass
-  - policy: resource-lists
-    rule: validate-image-tag
-    resources: 
-    - myapp-pod3
-    kind: Pod
-    result: pass 
+- kind: Pod
+  policy: resource-lists
+  resources:
+  - myapp-pod3
+  result: pass
+  rule: validate-image-tag
+- kind: Pod
+  policy: resource-lists
+  resources:
+  - myapp-pod1
+  - myapp-pod2
+  result: pass
+  rule: require-image-tag

test/cli/test/restrict-something/kyverno-test.yaml

@@ -1,19 +1,18 @@
-# Taken from https://github.com/kyverno/kyverno/issues/6463
 name: repro-dups-bug
 policies:
-  - policy.yaml
+- policy.yaml
 resources:
-  - resources.yaml
+- resources.yaml
 results:
-  - policy: restrict-something
-    rule: validate-some-foo
-    resources:
-    - nginx-foo
-    kind: Pod
-    result: pass
-  - policy: restrict-something
-    rule: validate-some-non-foo
-    resources:
-    - nginx-too
-    kind: Pod
-    result: fail
+- kind: Pod
+  policy: restrict-something
+  resources:
+  - nginx-too
+  result: fail
+  rule: validate-some-non-foo
+- kind: Pod
+  policy: restrict-something
+  resources:
+  - nginx-foo
+  result: pass
+  rule: validate-some-foo

test/cli/test/restrict_ingress_host/kyverno-test.yaml

@@ -4,12 +4,6 @@ policies:
 resources:
 - resource.yaml
 results:
-- kind: Ingress
-  policy: unique-ingress-host
-  resources:
-  - ingress-kyverno-host
-  result: fail
-  rule: check-single-host
 - kind: Ingress
   policy: unique-ingress-host
   resources:
@@ -20,12 +14,18 @@ results:
   policy: unique-ingress-host
   resources:
   - ingress-kyverno-host
-  result: skip
+  result: fail
+  rule: check-single-host
+- kind: Ingress
+  policy: unique-ingress-host
+  resources:
+  - ingress-foo-host
+  result: fail
   rule: deny-multiple-hosts
 - kind: Ingress
   policy: unique-ingress-host
   resources:
-  - ingress-foo-host
-  result: fail
+  - ingress-kyverno-host
+  result: skip
   rule: deny-multiple-hosts
 variables: values.yaml

test/cli/test/secret/kyverno-test.yaml

@@ -4,13 +4,6 @@ policies:
 resources:
 - resources.yaml
 results:
-- kind: Secret
-  patchedResource: patched-resource.yaml
-  policy: add-maintainer
-  resources:
-  - example
-  result: pass
-  rule: add-maintainer
 - kind: Secret
   patchedResource: patched-resource1.yaml
   policy: add-maintainer
@@ -18,3 +11,10 @@ results:
   - secrete-fail-example
   result: fail
   rule: add-maintainer
+- kind: Secret
+  patchedResource: patched-resource.yaml
+  policy: add-maintainer
+  resources:
+  - example
+  result: pass
+  rule: add-maintainer

test/cli/test/simple/kyverno-test.yaml

@@ -7,27 +7,9 @@ results:
 - kind: Pod
   policy: disallow-latest-tag
   resources:
-  - test/test-validate-image-tag-fail
+  - test/test-require-image-tag-fail
   result: fail
-  rule: validate-image-tag
-- kind: Pod
-  policy: duration-test
-  resources:
-  - test/test-lifetime-fail
-  result: fail
-  rule: greater-than
-- kind: Pod
-  policy: disallow-latest-tag
-  resources:
-  - test/test-validate-image-tag-pass
-  result: pass
-  rule: validate-image-tag
-- kind: Pod
-  policy: duration-test
-  resources:
-  - test/test-lifetime-fail
-  result: pass
-  rule: less-equal-than
+  rule: require-image-tag
 - kind: Pod
   policy: disallow-latest-tag
   resources:
@@ -37,30 +19,48 @@ results:
 - kind: Pod
   policy: disallow-latest-tag
   resources:
-  - test/test-require-image-tag-fail
+  - test/test-validate-image-tag-fail
   result: fail
-  rule: require-image-tag
+  rule: validate-image-tag
 - kind: Pod
-  policy: duration-test
+  policy: disallow-latest-tag
   resources:
-  - test/test-lifetime-fail
+  - test/test-validate-image-tag-pass
   result: pass
-  rule: less-than
+  rule: validate-image-tag
 - kind: Pod
   policy: duration-test
   resources:
   - test/test-lifetime-fail
   result: fail
   rule: greater-equal-than
+- kind: Pod
+  policy: duration-test
+  resources:
+  - test/test-lifetime-fail
+  result: fail
+  rule: greater-than
 - kind: Pod
   policy: restrict-pod-counts
   resources:
   - myapp-pod
   - test-validate-image-tag-ignore
-  - test/test-require-image-tag-pass
   - test/test-require-image-tag-fail
+  - test/test-require-image-tag-pass
   - test/test-validate-image-tag-fail
   - test/test-validate-image-tag-pass
   result: fail
   rule: restrict-pod-count
+- kind: Pod
+  policy: duration-test
+  resources:
+  - test/test-lifetime-fail
+  result: pass
+  rule: less-equal-than
+- kind: Pod
+  policy: duration-test
+  resources:
+  - test/test-lifetime-fail
+  result: pass
+  rule: less-than
 variables: values.yaml

test/cli/test/validating-admission-policies/disallow-host-path/kyverno-test.yaml

@@ -8,11 +8,11 @@ results:
   kind: Deployment
   policy: disallow-host-path
   resources:
-  - deployment-pass
-  result: pass
+  - deployment-fail
+  result: fail
 - isValidatingAdmissionPolicy: true
   kind: Deployment
   policy: disallow-host-path
   resources:
-  - deployment-fail
-  result: fail
+  - deployment-pass
+  result: pass

test/cli/test/variables/kyverno-test.yaml

@@ -10,30 +10,17 @@ resources:
 - resources.yaml
 results:
 - kind: Pod
-  policy: images
-  resources:
-  - test-pod-with-non-root-user-image
-  - test-pod-with-trusted-registry
-  result: pass
-  rule: only-allow-trusted-images
-- kind: Pod
-  policy: cm-variable-example
-  resources:
-  - test-env-test
-  result: pass
-  rule: example-configmap-lookup
-- kind: Pod
-  policy: cm-variable-example
+  policy: cm-multiple-example
   resources:
   - test-env-dev
   result: fail
   rule: example-configmap-lookup
 - kind: Pod
-  policy: cm-array-example
+  policy: cm-multiple-example
   resources:
-  - test-web
-  result: fail
-  rule: validate-role-annotation
+  - test-env-test
+  result: pass
+  rule: example-configmap-lookup
 - kind: Pod
   policy: cm-array-example
   resources:
@@ -43,32 +30,20 @@ results:
 - kind: Pod
   policy: cm-blk-scalar-example
   resources:
-  - test-blk-app
-  result: pass
+  - test-blk-web
+  result: fail
   rule: validate-blk-role-annotation
 - kind: Pod
-  policy: images
+  policy: cm-globalval-example
   resources:
-  - test-pod-with-non-trusted-registry
+  - test-global-prod
   result: fail
-  rule: only-allow-trusted-images
-- kind: Pod
-  policy: cm-multiple-example
-  resources:
-  - test-env-test
-  result: pass
-  rule: example-configmap-lookup
-- kind: Pod
-  policy: cm-multiple-example
-  resources:
-  - test-env-dev
-  result: fail
-  rule: example-configmap-lookup
+  rule: validate-mode
 - kind: Pod
   policy: cm-blk-scalar-example
   resources:
-  - test-blk-web
-  result: fail
+  - test-blk-app
+  result: pass
   rule: validate-blk-role-annotation
 - kind: Pod
   policy: cm-globalval-example
@@ -77,9 +52,34 @@ results:
   result: pass
   rule: validate-mode
 - kind: Pod
-  policy: cm-globalval-example
+  policy: cm-array-example
   resources:
-  - test-global-prod
+  - test-web
   result: fail
-  rule: validate-mode
+  rule: validate-role-annotation
+- kind: Pod
+  policy: cm-variable-example
+  resources:
+  - test-env-dev
+  result: fail
+  rule: example-configmap-lookup
+- kind: Pod
+  policy: images
+  resources:
+  - test-pod-with-non-trusted-registry
+  result: fail
+  rule: only-allow-trusted-images
+- kind: Pod
+  policy: cm-variable-example
+  resources:
+  - test-env-test
+  result: pass
+  rule: example-configmap-lookup
+- kind: Pod
+  policy: images
+  resources:
+  - test-pod-with-non-root-user-image
+  - test-pod-with-trusted-registry
+  result: pass
+  rule: only-allow-trusted-images
 variables: variables.yaml

test/cli/test/wildcard_match_label_selector/kyverno-test.yaml

@@ -4,35 +4,21 @@ policies:
 resources:
 - resources.yaml
 results:
-- kind: Pod
-  policy: wildcard-support-in-matchlabels
-  resources:
-  - my-service-1
-  result: pass
-  rule: wildcard-label
 - kind: Pod
   policy: wildcard-support-in-matchlabels
   resources:
   - my-service-2
   result: pass
   rule: label-end-with-test
-# TODO CEB FIX
-# - kind: Pod
-#   policy: wildcard-support-in-matchlabels
-#   resources:
-#   - my-service-3
-#   result: skip
-#   rule: label-end-with-test
 - kind: Pod
   policy: wildcard-support-in-matchlabels
   resources:
   - my-service-4
   result: pass
   rule: label-start-with-test
-# TODO CEB FIX
-# - kind: Pod
-#   policy: wildcard-support-in-matchlabels
-#   resources:
-#   - my-service-5
-#   result: skip
-#   rule: label-start-with-test
+- kind: Pod
+  policy: wildcard-support-in-matchlabels
+  resources:
+  - my-service-1
+  result: pass
+  rule: wildcard-label