fix reporting

Signed-off-by: Jim Bugwadia <jim@nirmata.com>

pkg/api/kyverno/v1/utils.go

@@ -199,6 +199,6 @@ type ViolatedRule struct {
 	// +optional
 	Message string `json:"message" yaml:"message"`
 
-	// +optional
-	Check string `json:"check" yaml:"check"`
+	// Status shows the rule response status
+	Status string `json:"status" yaml:"status"`
 }

pkg/kyverno/apply/report.go

@@ -108,7 +108,7 @@ func buildPolicyResults(infos []policyreport.Info) map[string][]*report.PolicyRe
 
 				result.Rule = rule.Name
 				result.Message = rule.Message
-				result.Result = report.PolicyResult(rule.Check)
+				result.Result = report.PolicyResult(rule.Status)
 				result.Source = policyreport.SourceValue
 				result.Timestamp = now
 				results[appname] = append(results[appname], &result)

pkg/kyverno/common/common.go

@@ -768,20 +768,36 @@ func ProcessValidateEngineResponse(policy *v1.ClusterPolicy, validateResponse *r
 					Message: valResponseRule.Message,
 				}
 
-				if valResponseRule.Status == response.RuleStatusPass {
+				switch valResponseRule.Status {
+				case response.RuleStatusPass:
 					rc.Pass++
-					vrule.Check = report.StatusPass
-				} else {
+					vrule.Status = report.StatusPass
+
+				case response.RuleStatusFail:
+					rc.Fail++
+					vrule.Status = report.StatusFail
 					if !policyReport {
 						if printCount < 1 {
 							fmt.Printf("\npolicy %s -> resource %s failed: \n", policy.Name, resPath)
 							printCount++
 						}
+
 						fmt.Printf("%d. %s: %s \n", i+1, valResponseRule.Name, valResponseRule.Message)
 					}
-					rc.Fail++
-					vrule.Check = report.StatusFail
+
+				case response.RuleStatusError:
+					rc.Error++
+					vrule.Status = report.StatusError
+
+				case response.RuleStatusWarn:
+					rc.Warn++
+					vrule.Status = report.StatusWarn
+
+				case response.RuleStatusSkip:
+					rc.Skip++
+					vrule.Status = report.StatusSkip
 				}
+
 				violatedRules = append(violatedRules, vrule)
 				continue
 			}
@@ -793,7 +809,7 @@ func ProcessValidateEngineResponse(policy *v1.ClusterPolicy, validateResponse *r
 				Name:    policyRule.Name,
 				Type:    "Validation",
 				Message: policyRule.Validation.Message,
-				Check:   report.StatusSkip,
+				Status:   report.StatusSkip,
 			}
 			violatedRules = append(violatedRules, vruleSkip)
 		}

pkg/kyverno/test/test_command.go

@@ -255,6 +255,7 @@ func buildPolicyResults(resps []*response.EngineResponse, testResults []TestResu
 					Name: resourceName,
 				},
 			},
+			Message: buildMessage(resp),
 		}
 
 		for i, test := range testResults {
@@ -297,7 +298,7 @@ func buildPolicyResults(resps []*response.EngineResponse, testResults []TestResu
 				}
 
 				result.Rule = rule.Name
-				result.Result = report.PolicyResult(rule.Check)
+				result.Result = report.PolicyResult(rule.Status)
 				result.Source = policyreport.SourceValue
 				result.Timestamp = now
 				results[resultsKey] = result
@@ -308,6 +309,16 @@ func buildPolicyResults(resps []*response.EngineResponse, testResults []TestResu
 	return results, testResults
 }
 
+func buildMessage(resp *response.EngineResponse) string {
+	var bldr strings.Builder
+	for _, ruleResp := range resp.PolicyResponse.Rules {
+		fmt.Fprintf(&bldr, "  %s: %s \n", ruleResp.Name, ruleResp.Status.String())
+		fmt.Fprintf(&bldr, "    %s \n", ruleResp.Message)
+	}
+
+	return bldr.String()
+}
+
 func getPolicyResourceFullPath(path []string, policyResourcePath string, isGit bool) []string {
 	var pol []string
 	if !isGit {
@@ -464,17 +475,20 @@ func printTestResult(resps map[string]report.PolicyReportResult, testResults []T
 			v.Result = v.Status
 		}
 		if testRes.Result == v.Result {
+			res.Result = boldGreen.Sprintf("Pass")
 			if testRes.Result == report.StatusSkip {
-				res.Result = boldGreen.Sprintf("Pass")
 				rc.Skip++
 			} else {
-				res.Result = boldGreen.Sprintf("Pass")
 				rc.Pass++
 			}
 		} else {
+			fmt.Printf("test failed for policy=%s, rule=%s, resource=%s, expected=%s, recieved=%s \n",
+				v.Policy, v.Rule, v.Resource, v.Result, testRes.Result)
+			fmt.Printf("%s \n", testRes.Message)
 			res.Result = boldRed.Sprintf("Fail")
 			rc.Fail++
 		}
+
 		table = append(table, res)
 	}
 	printer.BorderTop, printer.BorderBottom, printer.BorderLeft, printer.BorderRight = true, true, true, true

pkg/policyreport/builder.go

@@ -160,7 +160,7 @@ func (builder *requestBuilder) buildRCRResult(policy string, resource response.R
 
 	result.Rule = rule.Name
 	result.Message = rule.Message
-	result.Result = report.PolicyResult(rule.Check)
+	result.Result = report.PolicyResult(rule.Status)
 	if result.Result == "fail" && !av.scored {
 		result.Result = "warn"
 	}
@@ -263,15 +263,31 @@ func buildViolatedRules(er *response.EngineResponse) []kyverno.ViolatedRule {
 			Type:    rule.Type,
 			Message: rule.Message,
 		}
-		vrule.Check = report.StatusFail
-		if rule.Status == response.RuleStatusPass {
-			vrule.Check = report.StatusPass
-		}
+
+		vrule.Status = toPolicyResult(rule.Status)
 		violatedRules = append(violatedRules, vrule)
 	}
+
 	return violatedRules
 }
 
+func toPolicyResult(status response.RuleStatus) string {
+	switch status {
+	case response.RuleStatusPass:
+		return report.StatusPass
+	case response.RuleStatusFail:
+		return report.StatusFail
+	case response.RuleStatusError:
+		return report.StatusError
+	case response.RuleStatusWarn:
+		return report.StatusWarn
+	case response.RuleStatusSkip:
+		return report.StatusSkip
+	}
+
+	return ""
+}
+
 const categoryLabel string = "policies.kyverno.io/category"
 const severityLabel string = "policies.kyverno.io/severity"
 const scoredLabel string = "policies.kyverno.io/scored"