add success tests for validation & mutation

pkg/engine/overlay.go

@@ -76,9 +76,14 @@ func processOverlayNew(rule kyverno.Rule, resource unstructured.Unstructured) (r
 		return response, resource
 	}
 
-	joinedPatches := JoinPatches(patches)
 	var patchResource []byte
-	patchResource, err = ApplyPatchNew(resourceRaw, joinedPatches)
+	patchResource, err = ApplyPatches(resourceRaw, patches)
+	if err != nil {
+		glog.Info("failed to apply patch")
+		response.Success = false
+		response.Message = fmt.Sprintf("failed to apply JSON patches: %v", err)
+		return response, resource
+	}
 	err = patchedResource.UnmarshalJSON(patchResource)
 	if err != nil {
 		glog.Infof("failed to unmarshall resource to undstructured: %v", err)

pkg/engine/response.go

@@ -22,7 +22,7 @@ type PolicyResponse struct {
 	// resource details
 	Resource ResourceSpec `json:"resource"`
 	// policy statistics
-	PolicyStats
+	PolicyStats `json:",inline"`
 	// rule response
 	Rules []RuleResponse `json:"rules"`
 	// ValidationFailureAction: audit,enforce(default)
@@ -59,7 +59,7 @@ type RuleResponse struct {
 	// success/fail
 	Success bool `json:"success"`
 	// statistics
-	RuleStats
+	RuleStats `json:",inline"`
 }
 
 //ToString ...

pkg/engine/validation.go

@@ -380,7 +380,7 @@ func ValidateNew(policy kyverno.Policy, resource unstructured.Unstructured) (res
 			glog.V(4).Infof("resource %s/%s does not satisfy the resource description for the rule ", resource.GetNamespace(), resource.GetName())
 			continue
 		}
-		if rule.Validation.Pattern != nil {
+		if rule.Validation.Pattern != nil || rule.Validation.AnyPattern != nil {
 			ruleResponse := validatePatterns(resource, rule)
 			incrementAppliedRuleCount()
 			response.PolicyResponse.Rules = append(response.PolicyResponse.Rules, ruleResponse)

pkg/testrunner/scenario.go

@@ -132,40 +132,60 @@ func runTestCase(t *testing.T, tc scaseT) bool {
 	// generate mock client if resources are to be loaded
 	// - create mock client
 	// - load resources
-	client := getClient(t, tc.Input.LoadResources)
-	t.Log(client)
+	// client := getClient(t, tc.Input.LoadResources)
+
 	// apply policy
 	// convert policy -> kyverno.Policy
 	policy := loadPolicy(t, tc.Input.Policy)
-	fmt.Println(policy)
 	// convert resource -> unstructured.Unstructured
 	resource := loadPolicyResource(t, tc.Input.Resource)
-	glog.Info(resource)
 
 	var er engine.EngineResponseNew
 	// Mutation
 	er = engine.MutateNew(*policy, *resource)
 	func() {
 		if data, err := json.Marshal(er.PolicyResponse); err == nil {
+			t.Log("-----engine response----")
 			t.Log(string(data))
-			fmt.Println(string(data))
-			for _, r := range er.PolicyResponse.Rules {
-				for _, p := range r.Patches {
-					fmt.Println(string(p))
-				}
-			}
+			// for _, r := range er.PolicyResponse.Rules {
+			// 	for _, p := range r.Patches {
+			// 		fmt.Println(string(p))
+			// 	}
+			// }
 		}
 	}()
-
+	// validate te response
+	t.Log("---Mutation---")
 	validateResource(t, er.PatchedResource, tc.Expected.Mutation.PatchedResource)
 	validateResponse(t, er.PolicyResponse, tc.Expected.Mutation.PolicyResponse)
 
+	// pass the patched resource from mutate to validate
+	if len(er.PolicyResponse.Rules) > 0 {
+		resource = &er.PatchedResource
+	}
+
 	// Validation
-	// only compare the parametes specified ?
+	er = engine.ValidateNew(*policy, *resource)
+	func() {
+		if data, err := json.Marshal(er.PolicyResponse); err == nil {
+			t.Log(string(data))
+			fmt.Println(string(data))
+		}
+	}()
+	// validate the response
+	t.Log("---Validation---")
+	validateResponse(t, er.PolicyResponse, tc.Expected.Validation.PolicyResponse)
 	return true
 }
 
 func validateResource(t *testing.T, responseResource unstructured.Unstructured, expectedResourceFile string) {
+	resourcePrint := func(obj unstructured.Unstructured) {
+		t.Log("-----patched resource----")
+		if data, err := obj.MarshalJSON(); err == nil {
+			t.Log(string(data))
+		}
+	}
+	resourcePrint(responseResource)
 	if expectedResourceFile == "" {
 		t.Log("expected resource file not specified, wont compare resources")
 		return
@@ -176,13 +196,8 @@ func validateResource(t *testing.T, responseResource unstructured.Unstructured,
 		t.Log("failed to get the expected resource")
 		return
 	}
-	resourcePrint := func(obj unstructured.Unstructured) {
-		if data, err := obj.MarshalJSON(); err == nil {
-			fmt.Println(string(data))
-		}
-	}
-	resourcePrint(responseResource)
-	resourcePrint(*expectedResource)
+
+	// resourcePrint(*expectedResource)
 	// compare the resources
 	if !reflect.DeepEqual(responseResource, *expectedResource) {
 		t.Log("failed: response resource returned does not match expected resource")
@@ -191,22 +206,24 @@ func validateResource(t *testing.T, responseResource unstructured.Unstructured,
 }
 
 func validateResponse(t *testing.T, er engine.PolicyResponse, expected engine.PolicyResponse) {
+	if reflect.DeepEqual(expected, (engine.PolicyResponse{})) {
+		t.Log("no response expected")
+		return
+	}
 	// cant do deepEquals and the stats will be different, or we nil those fields and then do a comparison
 	// forcing only the fields that are specified to be comprared
 
 	// doing a field by fields comparsion will allow us to provied more details logs and granular error reporting
 	// check policy name is same :P
 	if er.Policy != expected.Policy {
-		t.Log("Policy: error")
+		t.Error("Policy: error")
 	}
 	// compare resource spec
-	if er.Resource != expected.Resource {
-		t.Log("Resource: error")
-	}
-	// stats
-	if er.RulesAppliedCount != expected.RulesAppliedCount {
-		t.Log("RulesAppliedCount: error")
-	}
+	compareResourceSpec(t, er.Resource, expected.Resource)
+	// // stats
+	// if er.RulesAppliedCount != expected.RulesAppliedCount {
+	// 	t.Log("RulesAppliedCount: error")
+	// }
 	// rules
 	if len(er.Rules) != len(er.Rules) {
 		t.Log("rule count: error")
@@ -225,10 +242,10 @@ func compareResourceSpec(t *testing.T, resource engine.ResourceSpec, expectedRes
 	if resource.Kind != expectedResource.Kind {
 		t.Error("error: kind")
 	}
-	// apiVersion
-	if resource.APIVersion != expectedResource.APIVersion {
-		t.Error("error: apiVersion")
-	}
+	// // apiVersion
+	// if resource.APIVersion != expectedResource.APIVersion {
+	// 	t.Error("error: apiVersion")
+	// }
 	// namespace
 	if resource.Namespace != expectedResource.Namespace {
 		t.Error("error: namespace")
@@ -242,17 +259,17 @@ func compareResourceSpec(t *testing.T, resource engine.ResourceSpec, expectedRes
 func compareRules(t *testing.T, rule engine.RuleResponse, expectedRule engine.RuleResponse) {
 	// name
 	if rule.Name != expectedRule.Name {
-		t.Logf("error rule %s: name", rule.Name)
+		t.Errorf("error rule %s: name", rule.Name)
 		// as the rule names dont match no need to compare the rest of the information
 		return
 	}
 	// type
 	if rule.Type != expectedRule.Type {
-		t.Log("error: typw")
+		t.Error("error: type")
 	}
 	// message
 	if rule.Message != expectedRule.Message {
-		t.Log("error: message")
+		t.Error("error: message")
 	}
 	// // patches
 	// if reflect.DeepEqual(rule.Patches, expectedRule.Patches) {
@@ -260,7 +277,7 @@ func compareRules(t *testing.T, rule engine.RuleResponse, expectedRule engine.Ru
 	// }
 	// success
 	if rule.Success != expectedRule.Success {
-		t.Log("error: success")
+		t.Error("error: success")
 	}
 	// statistics
 }
@@ -389,3 +406,13 @@ func loadPolicy(t *testing.T, path string) *kyverno.Policy {
 	}
 	return policies[0]
 }
+
+func testScenario(t *testing.T, path string) {
+	//load scenario
+	scenario, err := loadScenario(t, path)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	runScenario(t, scenario)
+}

pkg/testrunner/testrunner_test.go

@@ -8,11 +8,35 @@ import "testing"
 // }
 
 func Test_Devlop(t *testing.T) {
-	//load scenario
-	scenario, err := loadScenario(t, "/test/scenarios/test/s1.yaml")
-	if err != nil {
-		t.Error(err)
-	}
-	runScenario(t, scenario)
-
+	testScenario(t, "/test/scenarios/test/s1.yaml")
 }
+
+func Test_Mutate_EndPoint(t *testing.T) {
+	testScenario(t, "/test/scenarios/test/scenario_mutate_endPpoint.yaml")
+}
+
+func Test_Mutate_imagePullPolicy(t *testing.T) {
+	testScenario(t, "/test/scenarios/test/scenario_mutate_imagePullPolicy.yaml")
+}
+
+func Test_Mutate_Validate_qos(t *testing.T) {
+	testScenario(t, "/test/scenarios/test/scenario_mutate_validate_qos.yaml")
+}
+
+func Test_validate_containerSecurityContext(t *testing.T) {
+	testScenario(t, "/test/scenarios/test/scenario_validate_containerSecurityContext.yaml")
+}
+
+func Test_validate_healthChecks(t *testing.T) {
+	testScenario(t, "/test/scenarios/test/scenario_validate_healthChecks.yaml")
+}
+
+func Test_validate_imageRegistries(t *testing.T) {
+	testScenario(t, "/test/scenarios/test/scenario_validate_imageRegistries.yaml")
+}
+
+func Test_validate_nonRootUsers(t *testing.T) {
+	testScenario(t, "/test/scenarios/test/scenario_validate_nonRootUser.yaml")
+}
+
+//TODO add tests for Generation