Improved error handling for test command

Signed-off-by: Trey Dockendorf <tdockendorf@osc.edu>

Makefile

@@ -179,6 +179,7 @@ test-e2e:
 run_testcmd_policy:
 	go build -o  kyvernoctl cmd/cli/kubectl-kyverno/main.go
 	./kyvernoctl test https://github.com/kyverno/policies/main
+	./kyvernoctl test ./test/cli/test
 
 # godownloader create downloading script for kyverno-cli
 godownloader:

pkg/kyverno/test/command.go

@@ -145,36 +145,34 @@ func testCommandExecute(dirPath []string, valuesFile string, fileName string) (r
 		sort.Strings(policyYamls)
 		for _, yamlFilePath := range policyYamls {
 			file, err := fs.Open(yamlFilePath)
+			if err != nil {
+				errors = append(errors, sanitizederror.NewWithError("Error: failed to open file", err))
+				continue
+			}
 			if strings.Contains(file.Name(), fileName) {
 				testYamlCount++
 				policyresoucePath := strings.Trim(yamlFilePath, fileName)
 				bytes, err := ioutil.ReadAll(file)
 				if err != nil {
-					sanitizederror.NewWithError("Error: failed to read file", err)
+					errors = append(errors, sanitizederror.NewWithError("Error: failed to read file", err))
 					continue
 				}
 				policyBytes, err := yaml.ToJSON(bytes)
 				if err != nil {
-					sanitizederror.NewWithError("failed to convert to JSON", err)
+					errors = append(errors, sanitizederror.NewWithError("failed to convert to JSON", err))
 					continue
 				}
 				if err := applyPoliciesFromPath(fs, policyBytes, valuesFile, true, policyresoucePath, rc); err != nil {
 					return rc, sanitizederror.NewWithError("failed to apply test command", err)
 				}
 			}
-			if err != nil {
-				sanitizederror.NewWithError("Error: failed to open file", err)
-				continue
 		}
+		if testYamlCount == 0 {
+			fmt.Printf("\n No test yamls available \n")
 		}
 	} else {
 		path := filepath.Clean(dirPath[0])
-		if err != nil {
+		errors = getLocalDirTestFiles(fs, path, fileName, valuesFile, rc)
-			errors = append(errors, err)
-		}
-		err := getLocalDirTestFiles(fs, path, fileName, valuesFile, rc, testYamlCount)
-		if err != nil {
-			errors = append(errors, err)
 	}
 	if len(errors) > 0 && log.Log.V(1).Enabled() {
 		fmt.Printf("ignoring errors: \n")
@@ -182,46 +180,42 @@ func testCommandExecute(dirPath []string, valuesFile string, fileName string) (r
 			fmt.Printf("    %v \n", e.Error())
 		}
 	}
-	}
 	if rc.fail > 0 {
 		os.Exit(1)
 	}
-	if testYamlCount == 0 {
-		fmt.Printf("\n No test yamls available \n")
-	}
 	os.Exit(0)
 	return rc, nil
 }
 
-func getLocalDirTestFiles(fs billy.Filesystem, path, fileName, valuesFile string, rc *resultCounts, testYamlCount int) error {
+func getLocalDirTestFiles(fs billy.Filesystem, path, fileName, valuesFile string, rc *resultCounts) []error {
+	var errors []error
 	files, err := ioutil.ReadDir(path)
 	if err != nil {
-		return fmt.Errorf("failed to read %v: %v", path, err.Error())
+		return []error{fmt.Errorf("failed to read %v: %v", path, err.Error())}
 	}
 	for _, file := range files {
 		if file.IsDir() {
-			getLocalDirTestFiles(fs, filepath.Join(path, file.Name()), fileName, valuesFile, rc, testYamlCount)
+			getLocalDirTestFiles(fs, filepath.Join(path, file.Name()), fileName, valuesFile, rc)
 			continue
 		}
 		if strings.Contains(file.Name(), fileName) {
-			testYamlCount++
 			yamlFile, err := ioutil.ReadFile(filepath.Join(path, file.Name()))
 			if err != nil {
-				sanitizederror.NewWithError("unable to read yaml", err)
+				errors = append(errors, sanitizederror.NewWithError("unable to read yaml", err))
 				continue
 			}
 			valuesBytes, err := yaml.ToJSON(yamlFile)
 			if err != nil {
-				sanitizederror.NewWithError("failed to convert json", err)
+				errors = append(errors, sanitizederror.NewWithError("failed to convert json", err))
 				continue
 			}
 			if err := applyPoliciesFromPath(fs, valuesBytes, valuesFile, false, path, rc); err != nil {
-				sanitizederror.NewWithError("failed to apply test command", err)
+				errors = append(errors, sanitizederror.NewWithError(fmt.Sprintf("failed to apply test command from file %s", file.Name()), err))
 				continue
 			}
 		}
 	}
-	return nil
+	return errors
 }
 
 func buildPolicyResults(resps []*response.EngineResponse) map[string][]interface{} {

test/cli/test/policy.yaml

@@ -0,0 +1,35 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: disallow-latest-tag
+  annotations:
+    policies.kyverno.io/category: Best Practices
+    policies.kyverno.io/description: >-
+      The ':latest' tag is mutable and can lead to unexpected errors if the 
+      image changes. A best practice is to use an immutable tag that maps to 
+      a specific version of an application pod.      
+spec:
+  validationFailureAction: audit
+  rules:
+  - name: require-image-tag
+    match:
+      resources:
+        kinds:
+        - Pod
+    validate:
+      message: "An image tag is required."  
+      pattern:
+        spec:
+          containers:
+          - image: "*:*"
+  - name: validate-image-tag
+    match:
+      resources:
+        kinds:
+        - Pod
+    validate:
+      message: "Using a mutable image tag e.g. 'latest' is not allowed."
+      pattern:
+        spec:
+          containers:
+          - image: "!*:latest"

test/cli/test/resources.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-web
+  labels:
+    app: app
+spec: 
+  containers:
+  - name: nginx
+    image: nginx:latest
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-app
+  labels:
+    app: app
+spec: 
+  containers:
+  - name: nginx
+    image: nginx:1.12

test/cli/test/test.yaml

@@ -0,0 +1,14 @@
+name: test
+policies:
+  - policy.yaml
+resources:
+  - resources.yaml
+results:
+  - policy: disallow-latest-tag
+    rule: validate-image-tag
+    resource: test-web
+    status: fail
+  - policy: disallow-latest-tag
+    rule: validate-image-tag
+    resource: test-app
+    status: pass