update tests

Signed-off-by: Jim Bugwadia <jim@nirmata.com>

Makefile

@@ -148,21 +148,25 @@ create-e2e-infrastruture:
 
 ## variables
 BIN_DIR := $(GOPATH)/bin
-GO_ACC := $(BIN_DIR)/go-acc
+GO_ACC := $(BIN_DIR)/go-acc@latest
 CODE_COVERAGE_FILE:= coverage
 CODE_COVERAGE_FILE_TXT := $(CODE_COVERAGE_FILE).txt
 CODE_COVERAGE_FILE_HTML := $(CODE_COVERAGE_FILE).html
 
 ## targets
 $(GO_ACC):
-	@echo "	downloading testing tools"
-	go get -v github.com/ory/go-acc
+	@echo "	installing testing tools"
+	go install -v github.com/ory/go-acc@latest
 	$(eval export PATH=$(GO_ACC):$(PATH))
 # go test provides code coverage per packages only.
 # go-acc merges the result for pks so that it be used by
 # go tool cover for reporting
 
-test: test-unit test-e2e test-cmd
+test: test-clean test-unit test-e2e test-cmd
+
+test-clean:
+	@echo "	cleaning test cache"
+	go clean -testcache ./...
 
 
 # go get downloads and installs the binary

pkg/engine/mutate/strategicPreprocessing.go

@@ -325,7 +325,7 @@ func convertRNodeToInterface(document *yaml.RNode) (interface{}, error) {
 }
 
 func checkCondition(logger logr.Logger, pattern *yaml.RNode, resource *yaml.RNode) error {
-	patternInterface, err := convertRNodeToInterface(pattern)
+	patternInterface, err := convertRNodeToInterface(pattern);
 	if err != nil {
 		return err
 	}
@@ -335,8 +335,12 @@ func checkCondition(logger logr.Logger, pattern *yaml.RNode, resource *yaml.RNod
 		return err
 	}
 
-	err = validate.MatchPattern(logger, resourceInterface, patternInterface)
-	return err
+	err, _ = validate.MatchPattern(logger, resourceInterface, patternInterface)
+	if err != nil{
+		return err
+	}
+
+	return nil
 }
 
 func deleteConditionsFromNestedMaps(pattern *yaml.RNode) (bool, error) {

pkg/engine/mutate/strategicPreprocessing_test.go

@@ -913,7 +913,7 @@ func Test_CheckConditionAnchor_Matches(t *testing.T) {
 	resource := yaml.MustParse(string(resourceRaw))
 
 	err := checkCondition(log.Log, pattern, resource)
-	assert.NilError(t, err)
+	assert.Equal(t, err, nil)
 }
 
 func Test_CheckConditionAnchor_DoesNotMatch(t *testing.T) {

pkg/engine/response/response_test.go

@@ -0,0 +1,30 @@
+package response
+
+import (
+	"gopkg.in/yaml.v2"
+	"gotest.tools/assert"
+	"testing"
+)
+
+var sourceYAML = `
+policy:
+  name: disallow-bind-mounts
+resource:
+  kind: Pod
+  apiVersion: v1
+  name: image-with-hostpath
+rules:
+- name: validate-hostPath
+  type: Validation
+  status: Fail
+`
+
+func Test_parse_yaml(t *testing.T) {
+	var pr PolicyResponse
+	if err := yaml.Unmarshal([]byte(sourceYAML), &pr); err != nil {
+		t.Errorf("failed to parse YAML: %v", err)
+		return
+	}
+	assert.Equal(t, 1, len(pr.Rules))
+	assert.Equal(t, RuleStatusFail, pr.Rules[0].Status)
+}

pkg/engine/response/status.go

@@ -22,8 +22,8 @@ const (
 	RuleStatusSkip
 )
 
-func (s RuleStatus) String() string {
-	return toString[s]
+func (s *RuleStatus) String() string {
+	return toString[*s]
 }
 
 var toString = map[RuleStatus]string{
@@ -43,26 +43,50 @@ var toID = map[string]RuleStatus{
 }
 
 // MarshalJSON marshals the enum as a quoted json string
-func (s RuleStatus) MarshalJSON() ([]byte, error) {
+func (s *RuleStatus) MarshalJSON() ([]byte, error) {
 	var b strings.Builder
-	fmt.Fprintf(&b, "\"%s\"", toString[s])
+	fmt.Fprintf(&b, "\"%s\"", toString[*s])
 	return []byte(b.String()), nil
 }
 
 // UnmarshalJSON unmarshals a quoted json string to the enum value
 func (s *RuleStatus) UnmarshalJSON(b []byte) error {
-	var j string
-	err := json.Unmarshal(b, &j)
+	var strVal string
+	err := json.Unmarshal(b, &strVal)
 	if err != nil {
 		return err
 	}
 
+	statusVal, err := getRuleStatus(strVal)
+	if err != nil {
+		return err
+	}
+
+	*s = *statusVal
+	return nil
+}
+
+func getRuleStatus(s string) (*RuleStatus, error){
 	for k, v := range toID {
-		if j == k {
-			*s = v
-			return nil
+		if s == k {
+			return &v, nil
 		}
 	}
 
-	return fmt.Errorf("invalid status: %s", j)
+	return nil, fmt.Errorf("invalid status: %s", s)
+}
+
+func (v *RuleStatus) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+
+	statusVal, err := getRuleStatus(s)
+	if err != nil {
+		return err
+	}
+
+	*v = *statusVal
+	return nil
 }

pkg/engine/validate/common.go

@@ -41,11 +41,3 @@ func getRawKeyIfWrappedWithAttributes(str string) string {
 	}
 }
 
-type PatternError struct {
-	msg string
-	Path string
-}
-
-func (p* PatternError) Error() string {
-	return p.msg
-}

pkg/engine/validate/validate.go

@@ -14,25 +14,25 @@ import (
 
 // MatchPattern is a start of element-by-element pattern validation process.
 // It assumes that validation is started from root, so "/" is passed
-func MatchPattern(logger logr.Logger, resource, pattern interface{}) *PatternError {
+func MatchPattern(logger logr.Logger, resource, pattern interface{}) (error, string) {
 	// newAnchorMap - to check anchor key has values
 	ac := common.NewAnchorMap()
 	elemPath, err := validateResourceElement(logger, resource, pattern, pattern, "/", ac)
 	if err != nil {
 		if common.IsConditionalAnchorError(err.Error()) || common.IsGlobalAnchorError(err.Error()) {
 			logger.V(3).Info(ac.AnchorError.Message)
-			return &PatternError{ac.AnchorError.Message, ""}
+			return ac.AnchorError.Error(), ""
 		}
 
 		if ac.IsAnchorError() {
 			logger.V(3).Info("missing anchor in resource")
-			return &PatternError{err.Error(), ""}
+			return err, ""
 		}
 
-		return &PatternError{err.Error(), elemPath}
+		return err, elemPath
 	}
 
-	return nil
+	return nil, ""
 }
 
 // validateResourceElement detects the element type (map, array, nil, string, int, bool, float)

pkg/engine/validate/validate_test.go

@@ -1517,7 +1517,7 @@ func TestConditionalAnchorWithMultiplePatterns(t *testing.T) {
 		err = json.Unmarshal(testCase.resource, &resource)
 		assert.NilError(t, err)
 
-		_, err = MatchPattern(log.Log, resource, pattern)
+		err, _ = MatchPattern(log.Log, resource, pattern)
 		if testCase.nilErr {
 			assert.NilError(t, err, fmt.Sprintf("\ntest: %s\npattern: %s\nresource: %s\n", testCase.name, pattern, resource))
 		} else {

pkg/engine/validation.go

@@ -404,13 +404,13 @@ func isSameRuleResponse(r1 *response.RuleResponse, r2 *response.RuleResponse) bo
 // validatePatterns validate pattern and anyPattern
 func (v *validator) validatePatterns(resource unstructured.Unstructured) *response.RuleResponse {
 	if v.pattern != nil {
-		if err := validate.MatchPattern(v.log, resource.Object, v.pattern); err != nil {
-			v.log.V(3).Info("validation error", "path", err.Path, "error", err.Error())
-			if err.Path == "" {
+		if err, path := validate.MatchPattern(v.log, resource.Object, v.pattern); err != nil {
+			v.log.V(3).Info("validation error", "path", path, "error", err.Error())
+			if path == "" {
 				return ruleResponse(v.rule, v.buildErrorMessage(err, ""), response.RuleStatusError)
 			}
 
-			return ruleResponse(v.rule, v.buildErrorMessage(err, err.Path), response.RuleStatusFail)
+			return ruleResponse(v.rule, v.buildErrorMessage(err, path), response.RuleStatusFail)
 		}
 
 		v.log.V(4).Info("successfully processed rule")
@@ -429,18 +429,18 @@ func (v *validator) validatePatterns(resource unstructured.Unstructured) *respon
 		}
 
 		for idx, pattern := range anyPatterns {
-			err := validate.MatchPattern(v.log, resource.Object, pattern)
+			err, path := validate.MatchPattern(v.log, resource.Object, pattern)
 			if err == nil {
 				msg := fmt.Sprintf("validation rule '%s' anyPattern[%d] passed.", v.rule.Name, idx)
 				return ruleResponse(v.rule, msg, response.RuleStatusPass)
 			}
 
-			v.log.V(3).Info("validation rule failed", "anyPattern[%d]", idx, "path", err.Path)
-			if err.Path == "" {
+			v.log.V(3).Info("validation rule failed", "anyPattern[%d]", idx, "path", path)
+			if path == "" {
 				patternErr := fmt.Errorf("Rule %s[%d] failed: %s.", v.rule.Name, idx, err.Error())
 				failedAnyPatternsErrors = append(failedAnyPatternsErrors, patternErr)
 			} else {
-				patternErr := fmt.Errorf("Rule %s[%d] failed at path %s.", v.rule.Name, idx, err.Path)
+				patternErr := fmt.Errorf("Rule %s[%d] failed at path %s.", v.rule.Name, idx, path)
 				failedAnyPatternsErrors = append(failedAnyPatternsErrors, patternErr)
 			}
 		}

pkg/testrunner/scenario.go

@@ -3,9 +3,11 @@ package testrunner
 import (
 	"bytes"
 	"encoding/json"
+	"github.com/stretchr/testify/assert"
 	"io/ioutil"
 	"os"
 	ospath "path"
+	"path/filepath"
 	"reflect"
 	"testing"
 
@@ -14,83 +16,94 @@ import (
 	"github.com/kyverno/kyverno/pkg/engine"
 	"github.com/kyverno/kyverno/pkg/engine/context"
 	"github.com/kyverno/kyverno/pkg/engine/response"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
+	k8sRuntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	apiyaml "k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/kubernetes/scheme"
+	"path"
+	"runtime"
 )
 
-type scenarioT struct {
-	testCases []scaseT
+type Scenario struct {
+	TestCases []TestCase
 }
 
-//scase defines input and output for a case
-type scaseT struct {
-	Input    sInput    `yaml:"input"`
-	Expected sExpected `yaml:"expected"`
+//CaseT defines input and output for a case
+type TestCase struct {
+	Input    Input    `yaml:"input"`
+	Expected Expected `yaml:"expected"`
 }
 
-//sInput defines input for a test scenario
-type sInput struct {
+//Input defines input for a test scenario
+type Input struct {
 	Policy        string   `yaml:"policy"`
 	Resource      string   `yaml:"resource"`
 	LoadResources []string `yaml:"loadresources,omitempty"`
 }
 
-type sExpected struct {
-	Mutation   sMutation   `yaml:"mutation,omitempty"`
-	Validation sValidation `yaml:"validation,omitempty"`
-	Generation sGeneration `yaml:"generation,omitempty"`
+type Expected struct {
+	Mutation   Mutation   `yaml:"mutation,omitempty"`
+	Validation Validation `yaml:"validation,omitempty"`
+	Generation Generation `yaml:"generation,omitempty"`
 }
 
-type sMutation struct {
+type Mutation struct {
 	// path to the patched resource to be compared with
 	PatchedResource string `yaml:"patchedresource,omitempty"`
 	// expected response from the policy engine
 	PolicyResponse response.PolicyResponse `yaml:"policyresponse"`
 }
 
-type sValidation struct {
+type Validation struct {
 	// expected response from the policy engine
 	PolicyResponse response.PolicyResponse `yaml:"policyresponse"`
 }
 
-type sGeneration struct {
+type Generation struct {
 	// generated resources
 	GeneratedResources []kyverno.ResourceSpec `yaml:"generatedResources"`
 	// expected response from the policy engine
 	PolicyResponse response.PolicyResponse `yaml:"policyresponse"`
 }
 
-//getRelativePath expects a path relative to project and builds the complete path
-func getRelativePath(path string) string {
-	gp := os.Getenv("GOPATH")
-	ap := ospath.Join(gp, projectPath)
-	return ospath.Join(ap, path)
+// RootDir returns the kyverno project directory based on the location of the current file.
+// It assumes that the project directory is 2 levels up. This means if this function is moved
+// it may not work as expected.
+func RootDir() string {
+	_, b, _, _ := runtime.Caller(0)
+	d := path.Join(path.Dir(b))
+	d = filepath.Dir(d)
+	return filepath.Dir(d)
 }
 
-func loadScenario(t *testing.T, path string) (*scenarioT, error) {
-	fileBytes, err := loadFile(t, path)
-	if err != nil {
-		return nil, err
-	}
+//getRelativePath expects a path relative to project and builds the complete path
+func getRelativePath(path string) string {
+	root := RootDir()
+	return ospath.Join(root, path)
+}
 
-	var testCases []scaseT
+func loadScenario(t *testing.T, path string) (*Scenario, error) {
+	fileBytes, err := loadFile(t, path)
+	assert.Nil(t, err)
+
+	var testCases []TestCase
 	// load test cases separated by '---'
 	// each test case defines an input & expected result
 	scenariosBytes := bytes.Split(fileBytes, []byte("---"))
-	for _, scenarioBytes := range scenariosBytes {
-		tc := scaseT{}
-		if err := yaml.Unmarshal([]byte(scenarioBytes), &tc); err != nil {
+	for _, testCaseBytes := range scenariosBytes {
+		var tc TestCase
+		if err := yaml.Unmarshal(testCaseBytes, &tc); err != nil {
 			t.Errorf("failed to decode test case YAML: %v", err)
 			continue
 		}
+
 		testCases = append(testCases, tc)
 	}
-	scenario := scenarioT{
-		testCases: testCases,
+
+	scenario := Scenario{
+		TestCases: testCases,
 	}
 
 	return &scenario, nil
@@ -106,14 +119,14 @@ func loadFile(t *testing.T, path string) ([]byte, error) {
 	return ioutil.ReadFile(path)
 }
 
-func runScenario(t *testing.T, s *scenarioT) bool {
-	for _, tc := range s.testCases {
+func runScenario(t *testing.T, s *Scenario) bool {
+	for _, tc := range s.TestCases {
 		runTestCase(t, tc)
 	}
 	return true
 }
 
-func runTestCase(t *testing.T, tc scaseT) bool {
+func runTestCase(t *testing.T, tc TestCase) bool {
 	policy := loadPolicy(t, tc.Input.Policy)
 	if policy == nil {
 		t.Error("Policy not loaded")
@@ -311,7 +324,7 @@ func compareRules(t *testing.T, rule response.RuleResponse, expectedRule respons
 
 	// success
 	if rule.Status != expectedRule.Status {
-		t.Errorf("rule success: expected %v, received %v", expectedRule.Status, rule.Status)
+		t.Errorf("rule status mismatch: expected %s, received %s", expectedRule.Status.String(), rule.Status.String())
 	}
 }
 
@@ -330,7 +343,7 @@ func loadPolicyResource(t *testing.T, file string) *unstructured.Unstructured {
 }
 
 func getClient(t *testing.T, files []string) *client.Client {
-	var objects []runtime.Object
+	var objects []k8sRuntime.Object
 	if files != nil {
 
 		for _, file := range files {
@@ -338,7 +351,7 @@ func getClient(t *testing.T, files []string) *client.Client {
 		}
 	}
 	// create mock client
-	scheme := runtime.NewScheme()
+	scheme := k8sRuntime.NewScheme()
 	// mock client expects the resource to be as runtime.Object
 	c, err := client.NewMockClient(scheme, nil, objects...)
 	if err != nil {
@@ -352,7 +365,7 @@ func getClient(t *testing.T, files []string) *client.Client {
 	return c
 }
 
-func getGVRForResources(objects []runtime.Object) []schema.GroupVersionResource {
+func getGVRForResources(objects []k8sRuntime.Object) []schema.GroupVersionResource {
 	var gvrs []schema.GroupVersionResource
 	for _, obj := range objects {
 		gvk := obj.GetObjectKind().GroupVersionKind()
@@ -380,7 +393,7 @@ func loadResource(t *testing.T, path string) []*unstructured.Unstructured {
 			continue
 		}
 
-		data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&obj)
+		data, err := k8sRuntime.DefaultUnstructuredConverter.ToUnstructured(&obj)
 		if err != nil {
 			t.Logf("failed to unmarshall resource. %v", err)
 			continue
@@ -392,8 +405,8 @@ func loadResource(t *testing.T, path string) []*unstructured.Unstructured {
 	return unstrResources
 }
 
-func loadObjects(t *testing.T, path string) []runtime.Object {
-	var resources []runtime.Object
+func loadObjects(t *testing.T, path string) []k8sRuntime.Object {
+	var resources []k8sRuntime.Object
 	t.Logf("loading objects from %s", path)
 	data, err := loadFile(t, path)
 	if err != nil {

pkg/testrunner/scenario_test.go

@@ -0,0 +1,70 @@
+package testrunner
+
+import (
+	"github.com/kyverno/kyverno/pkg/engine/response"
+	"gopkg.in/yaml.v3"
+	"gotest.tools/assert"
+	"io/ioutil"
+	"testing"
+)
+
+var sourceYAML = `
+input:
+  policy: test/best_practices/disallow_bind_mounts.yaml
+  resource: test/resources/disallow_host_filesystem.yaml
+expected:
+  validation:
+    policyresponse:
+      policy:
+        namespace: ''
+        name: disallow-bind-mounts
+      resource:
+        kind: Pod
+        apiVersion: v1
+        namespace: ''
+        name: image-with-hostpath
+      rules:
+        - name: validate-hostPath
+          type: Validation
+          status: Fail
+`
+
+func Test_parse_yaml(t *testing.T) {
+	var s TestCase
+	if err := yaml.Unmarshal([]byte(sourceYAML), &s); err != nil {
+		t.Errorf("failed to parse YAML: %v", err)
+		return
+	}
+
+	assert.Equal(t, s.Expected.Validation.PolicyResponse.Policy.Name, "disallow-bind-mounts")
+	assert.Equal(t, 1, len(s.Expected.Validation.PolicyResponse.Rules), "invalid rule count")
+	assert.Equal(t, response.RuleStatusFail, s.Expected.Validation.PolicyResponse.Rules[0].Status, "invalid status")
+}
+
+func Test_parse_file(t *testing.T) {
+	s, err := loadScenario(t, "test/scenarios/samples/best_practices/disallow_bind_mounts_fail.yaml")
+	assert.NilError(t, err)
+
+	assert.Equal(t, 1, len(s.TestCases))
+	assert.Equal(t, s.TestCases[0].Expected.Validation.PolicyResponse.Policy.Name, "disallow-bind-mounts")
+	assert.Equal(t, 1, len(s.TestCases[0].Expected.Validation.PolicyResponse.Rules), "invalid rule count")
+	assert.Equal(t, response.RuleStatusFail, s.TestCases[0].Expected.Validation.PolicyResponse.Rules[0].Status, "invalid status")
+}
+
+
+func Test_parse_file2(t *testing.T) {
+	path := getRelativePath("test/scenarios/samples/best_practices/disallow_bind_mounts_fail.yaml")
+	data, err := ioutil.ReadFile(path)
+	assert.NilError(t, err)
+
+	strData := string(data)
+	var s TestCase
+	if err := yaml.Unmarshal([]byte(strData), &s); err != nil {
+		t.Errorf("failed to parse YAML: %v", err)
+		return
+	}
+
+	assert.Equal(t, s.Expected.Validation.PolicyResponse.Policy.Name, "disallow-bind-mounts")
+	assert.Equal(t, 1, len(s.Expected.Validation.PolicyResponse.Rules), "invalid rule count")
+	assert.Equal(t, response.RuleStatusFail, s.Expected.Validation.PolicyResponse.Rules[0].Status, "invalid status")
+}

pkg/testrunner/utils.go

@@ -8,10 +8,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
-var (
-	projectPath = envOr("PROJECT_PATH", "src/github.com/kyverno/kyverno")
-)
-
 // LoadFile loads file in byte buffer
 func LoadFile(path string) ([]byte, error) {
 	if _, err := os.Stat(path); os.IsNotExist(err) {

test/cli/test-fail/missing-rule/test.yaml

@@ -7,4 +7,4 @@ results:
   - policy: disallow-latest-tag
     rule: missing
     resource: test
-    status: pass
+    status: Pass

test/cli/test/simple/test.yaml

@@ -7,11 +7,11 @@ results:
   - policy: disallow-latest-tag
     rule: require-image-tag
     resource: test-require-image-tag-pass
-    status: pass
+    status: Pass
   - policy: disallow-latest-tag
     rule: require-image-tag
     resource: test-require-image-tag-fail
-    status: fail
+    status: Fail
   - policy: disallow-latest-tag
     rule: validate-image-tag
     resource: test-validate-image-tag-ignore
@@ -19,8 +19,8 @@ results:
   - policy: disallow-latest-tag
     rule: validate-image-tag
     resource: test-validate-image-tag-fail
-    status: fail
+    status: Fail
   - policy: disallow-latest-tag
     rule: validate-image-tag
     resource: test-validate-image-tag-pass
-    status: pass
+    status: Pass

test/e2e/mutate/mutate_test.go

@@ -216,7 +216,7 @@ func Test_Mutate(t *testing.T) {
 		Expect(err).NotTo(HaveOccurred())
 
 		By("Validating created resource with the expected pattern...")
-		_, err = validate.MatchPattern(log.Log, actual, expected)
+		err, _ = validate.MatchPattern(log.Log, actual, expected)
 		Expect(err).NotTo(HaveOccurred())
 
 		By("Deleting Cluster Policies...")

test/scenarios/other/scenario_mutate_endpoint.yaml

@@ -17,5 +17,5 @@ expected:
       rules:
         - name: pEP
           type: Mutation
-          success: true
+          status: Pass
           message: successfully process JSON patches

test/scenarios/other/scenario_mutate_pod_spec.yaml

@@ -16,5 +16,5 @@ expected:
       rules:
         - name: disable-servicelink-and-token
           type: Mutation
-          success: true
+          status: Pass
           message: successfully processed strategic merge patch

test/scenarios/other/scenario_mutate_validate_qos.yaml

@@ -17,7 +17,7 @@ expected:
       rules:
         - name: add-memory-limit
           type: Mutation
-          success: true
+          status: Pass
           message: successfully processed strategic merge patch
   validation:
     policyresponse:
@@ -33,4 +33,4 @@ expected:
         - name: check-cpu-memory-limits
           type: Validation
           message: validation rule 'check-cpu-memory-limits' passed.
-          success: true     
+          status: Pass

test/scenarios/other/scenario_validate_default_proc_mount.yaml

@@ -18,4 +18,4 @@ expected:
         - name: validate-default-proc-mount
           type: Validation
           message: "validation rule 'validate-default-proc-mount' passed."
-          success: true
+          status: Pass

test/scenarios/other/scenario_validate_disallow_default_serviceaccount.yaml

@@ -17,4 +17,4 @@ expected:
         - name: prevent-mounting-default-serviceaccount
           type: Validation
           message: "validation error: Prevent mounting of default service account. Rule prevent-mounting-default-serviceaccount failed at path /spec/serviceAccountName/"
-          success: false
+          status: Fail

test/scenarios/other/scenario_validate_healthChecks.yaml

@@ -17,8 +17,8 @@ expected:
         - name: check-readinessProbe-exists
           type: Validation
           message: validation rule 'check-readinessProbe-exists' passed.
-          success: true     
+          status: Pass
         - name: check-livenessProbe-exists
           type: Validation
           message: validation rule 'check-livenessProbe-exists' passed.
-          success: true     
+          status: Pass

test/scenarios/other/scenario_validate_selinux_context.yaml

@@ -17,4 +17,4 @@ expected:
         - name: validate-selinux-options
           type: Validation
           message: "validation error: SELinux level is required. Rule validate-selinux-options failed at path /spec/containers/0/securityContext/seLinuxOptions/"
-          success: false
+          status: Fail

test/scenarios/other/scenario_validate_volume_whiltelist.yaml

@@ -18,4 +18,4 @@ expected:
         - name: validate-volumes-whitelist
           type: Validation
           message: "validation rule 'validate-volumes-whitelist' anyPattern[2] passed."
-          success: true
+          status: Pass

test/scenarios/samples/best_practices/add_networkPolicy.yaml

@@ -20,5 +20,5 @@ expected:
       rules:
         - name: default-deny-ingress
           type: Generation
-          success: true
+          status: Pass
           message: created resource NetworkPolicy/devtest/default-deny-ingress

test/scenarios/samples/best_practices/add_ns_quota.yaml

@@ -20,7 +20,7 @@ expected:
       rules:
         - name: generate-resourcequota
           type: Generation
-          success: true
+          status: Pass
         - name: generate-limitrange
           type: Generation
-          success: true
+          status: Pass

test/scenarios/samples/best_practices/add_safe_to_evict.yaml

@@ -17,5 +17,5 @@ expected:
       rules:
         - name: annotate-empty-dir
           type: Mutation
-          success: true
+          status: Pass
           message: "successfully processed strategic merge patch"

test/scenarios/samples/best_practices/add_safe_to_evict2.yaml

@@ -17,5 +17,5 @@ expected:
       rules:
         - name: annotate-host-path
           type: Mutation
-          success: true
+          status: Pass
           message: "successfully processed strategic merge patch"

test/scenarios/samples/best_practices/disallow_bind_mounts_fail.yaml

@@ -15,5 +15,6 @@ expected:
         name: image-with-hostpath
       rules:
         - name: validate-hostPath
+          message: "validation error: Host path volumes are not allowed. Rule validate-hostPath failed at path /spec/volumes/0/hostPath/"
           type: Validation
-          success: false
+          status: Fail

test/scenarios/samples/best_practices/disallow_bind_mounts_pass.yaml

@@ -16,4 +16,4 @@ expected:
       rules:
         - name: validate-hostPath
           type: Validation
-          success: true
+          status: Pass

test/scenarios/samples/best_practices/disallow_host_network_port.yaml

@@ -16,7 +16,7 @@ expected:
       rules:
         - name: validate-host-network
           type: Validation
-          success: true   
+          status: Pass
         - name: validate-host-port
           type: Validation
-          success: false             
+          status: Fail

test/scenarios/samples/best_practices/disallow_host_pid_ipc.yaml

@@ -16,4 +16,4 @@ expected:
       rules:
         - name: validate-hostPID-hostIPC
           type: Validation
-          success: false
+          status: Fail

test/scenarios/samples/best_practices/disallow_priviledged.yaml

@@ -16,7 +16,7 @@ expected:
       rules:
         - name: validate-privileged
           type: Validation
-          success: false
+          status: Fail
         - name: validate-allowPrivilegeEscalation
           type: Validation
-          success: false
+          status: Fail

test/scenarios/samples/best_practices/disallow_sysctls.yaml

@@ -17,4 +17,4 @@ expected:
       rules:
         - name: validate-sysctls
           type: Validation
-          success: false
+          status: Fail

test/scenarios/samples/more/restrict_automount_sa_token.yaml

@@ -16,4 +16,4 @@ expected:
       rules:
         - name: validate-automountServiceAccountToken
           type: Validation
-          success: true 
+          status: Pass

test/scenarios/samples/more/restrict_ingress_classes.yaml

@@ -16,4 +16,4 @@ expected:
       rules:
         - name: validate-ingress
           type: Validation
-          success: true
+          status: Pass

test/scenarios/samples/more/unknown_ingress_class.yaml

@@ -16,4 +16,4 @@ expected:
       rules:
         - name: validate-ingress
           type: Validation
-          success: false
+          status: Fail