refactor testrunner framework

examples/cli/ghost.yaml

@@ -29,6 +29,7 @@ spec:
       containers:
       - name: "ghost"
         image: "nginx:latest"
+        imagePullPolicy: Always
         ports:
         - containerPort: 8080
           protocol: "TCP"

examples/generate/policy_basic.yaml

@@ -4,7 +4,7 @@ metadata :
   name : basic-policy
 spec :
   rules:
-    - name: "Basic config generator for all namespaces"
+    - name: "Basic clone config generator for all namespaces"
       resource:
         kinds: 
         - Namespace

pkg/engine/patches.go

@@ -16,10 +16,14 @@ type PatchBytes []byte
 // ProcessPatches Returns array from separate patches that can be applied to the document
 // Returns error ONLY in case when creation of resource should be denied.
 func ProcessPatches(rule kubepolicy.Rule, resource []byte) (allPatches []PatchBytes, errs []error) {
-	if len(resource) == 0 || rule.Mutation == nil {
+	if len(resource) == 0 {
 		errs = append(errs, errors.New("Source document for patching is empty"))
 		return nil, errs
 	}
+	if rule.Mutation == nil {
+		errs = append(errs, errors.New("No Mutation rules defined"))
+		return nil, errs
+	}
 	patchedDocument := resource
 	for _, patch := range rule.Mutation.Patches {
 		patchRaw, err := json.Marshal(patch)

pkg/testrunner/test.go

@@ -1,7 +1,7 @@
 package testrunner
 
 import (
-	"fmt"
+	"strconv"
 	"testing"
 
 	ospath "path"
@@ -10,7 +10,7 @@ import (
 	pt "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1"
 	client "github.com/nirmata/kyverno/pkg/dclient"
 	"github.com/nirmata/kyverno/pkg/engine"
-	"github.com/nirmata/kyverno/pkg/result"
+	"github.com/nirmata/kyverno/pkg/info"
 	kscheme "k8s.io/client-go/kubernetes/scheme"
 )
 
@@ -48,56 +48,102 @@ func (t *test) run() {
 
 	}
 	// apply the policy engine
-	pr, mResult, vResult, err := t.applyPolicy(t.policy, t.tResource, client)
+	pr, policyInfo, err := t.applyPolicy(t.policy, t.tResource, client)
 	if err != nil {
 		t.t.Error(err)
 		return
 	}
 	// Expected Result
-	t.checkMutationResult(pr, mResult)
-	t.checkValidationResult(vResult)
-	t.checkGenerationResult(client)
+	// Test succesfuly ?
+	t.overAllPass(policyInfo.IsSuccessful(), t.testCase.Expected.Passes)
+	t.checkMutationResult(pr, policyInfo)
+	t.checkValidationResult(policyInfo)
+	t.checkGenerationResult(client, policyInfo)
 }
 
-func (t *test) checkMutationResult(pr *resourceInfo, result result.Result) {
+func (t *test) checkMutationResult(pr *resourceInfo, policyInfo *info.PolicyInfo) {
 	if t.testCase.Expected.Mutation == nil {
 		glog.Info("No Mutation check defined")
 		return
 	}
 	// patched resource
 	if !compareResource(pr, t.patchedResource) {
-		fmt.Printf("Expected Resource %s \n", string(t.patchedResource.rawResource))
-		fmt.Printf("Patched Resource %s \n", string(pr.rawResource))
 		glog.Warningf("Expected resource %s ", string(pr.rawResource))
 		t.t.Error("Patched resources not as expected")
 	}
-	// reason
-	reason := t.testCase.Expected.Mutation.Reason
-	if len(reason) > 0 && result.GetReason().String() != reason {
-		t.t.Error("Reason not matching")
+
+	// check if rules match
+	t.compareRules(policyInfo.Rules, t.testCase.Expected.Mutation.Rules)
+}
+
+func (t *test) overAllPass(result bool, expected string) {
+	b, err := strconv.ParseBool(expected)
+	if err != nil {
+		t.t.Error(err)
+	}
+	if result != b {
+		t.t.Errorf("Expected value %v and actual value %v dont match", expected, result)
 	}
 }
 
-func (t *test) checkValidationResult(result result.Result) {
+func (t *test) compareRules(ruleInfos []*info.RuleInfo, rules []tRules) {
+	// Compare the rules specified in the expected against the actual rule info returned by the apply policy
+	for _, eRule := range rules {
+		// Look-up the rule from the policy info
+		rule := lookUpRule(eRule.Name, ruleInfos)
+		if rule == nil {
+			t.t.Errorf("Rule with name %s not found", eRule.Name)
+			continue
+		}
+		// get the corresponding rule
+		if rule.Name != eRule.Name {
+			t.t.Errorf("Rule Name not matching!. expected %s , actual %s", eRule.Name, rule.Name)
+		}
+		if rule.RuleType.String() != eRule.Type {
+			t.t.Errorf("Rule type mismatch!. expected %s, actual %s", eRule.Type, rule.RuleType.String())
+		}
+		if len(eRule.Messages) != len(rule.Msgs) {
+			t.t.Errorf("Number of rule messages not same. expected %d, actual %d", len(eRule.Messages), len(rule.Msgs))
+		}
+		for i, msg := range eRule.Messages {
+			if msg != rule.Msgs[i] {
+				t.t.Errorf("Messges dont match!. expected %s, actual %s", msg, rule.Msgs[i])
+			}
+		}
+	}
+}
+
+func lookUpRule(name string, ruleInfos []*info.RuleInfo) *info.RuleInfo {
+	for _, r := range ruleInfos {
+		if r.Name == name {
+			return r
+		}
+	}
+	return nil
+}
+
+func (t *test) checkValidationResult(policyInfo *info.PolicyInfo) {
 	if t.testCase.Expected.Validation == nil {
 		glog.Info("No Validation check defined")
 		return
 	}
-	// reason
-	reason := t.testCase.Expected.Validation.Reason
-	if len(reason) > 0 && result.GetReason().String() != reason {
-		t.t.Error("Reason not matching")
-	}
+
+	// check if rules match
+	t.compareRules(policyInfo.Rules, t.testCase.Expected.Validation.Rules)
 }
 
-func (t *test) checkGenerationResult(client *client.Client) {
+func (t *test) checkGenerationResult(client *client.Client, policyInfo *info.PolicyInfo) {
 	if t.testCase.Expected.Generation == nil {
 		glog.Info("No Generate check defined")
 		return
 	}
 	if client == nil {
-		glog.Info("client needs to be configured")
+		t.t.Error("client needs to be configured")
 	}
+
+	// check if rules match
+	t.compareRules(policyInfo.Rules, t.testCase.Expected.Generation.Rules)
+
 	// check if the expected resources are generated
 	for _, r := range t.genResources {
 		n := ParseNameFromObject(r.rawResource)
@@ -113,33 +159,51 @@ func (t *test) checkGenerationResult(client *client.Client) {
 
 func (t *test) applyPolicy(policy *pt.Policy,
 	tresource *resourceInfo,
-	client *client.Client) (*resourceInfo, result.Result, result.Result, error) {
+	client *client.Client) (*resourceInfo, *info.PolicyInfo, error) {
 	// apply policy on the trigger resource
 	// Mutate
-	var vResult result.Result
-	var patchedResource []byte
-	mPatches, mResult := engine.Mutate(*policy, tresource.rawResource, *tresource.gvk)
+	var err error
+	rawResource := tresource.rawResource
+	rname := engine.ParseNameFromObject(rawResource)
+	rns := engine.ParseNamespaceFromObject(rawResource)
+	rkind := engine.ParseKindFromObject(rawResource)
+	policyInfo := info.NewPolicyInfo(policy.Name,
+		rkind,
+		rname,
+		rns)
+	// Apply Mutation Rules
+	patches, ruleInfos := engine.Mutate(*policy, rawResource, *tresource.gvk)
+	policyInfo.AddRuleInfos(ruleInfos)
 	// TODO: only validate if there are no errors in mutate, why?
-	err := mResult.ToError()
-	if err == nil && len(mPatches) != 0 {
-		patchedResource, err = engine.ApplyPatches(tresource.rawResource, mPatches)
-		if err != nil {
-			return nil, nil, nil, err
+	if policyInfo.IsSuccessful() {
+		if len(patches) != 0 {
+			rawResource, err = engine.ApplyPatches(rawResource, patches)
+			if err != nil {
+				return nil, nil, err
+			}
+		}
+	}
+	// Validate
+	ruleInfos, err = engine.Validate(*policy, rawResource, *tresource.gvk)
+	policyInfo.AddRuleInfos(ruleInfos)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if rkind == "Namespace" {
+		if client != nil {
+			ruleInfos := engine.Generate(client, *policy, rawResource, *tresource.gvk, false)
+			policyInfo.AddRuleInfos(ruleInfos)
 		}
-		// Validate
-		vResult = engine.Validate(*policy, patchedResource, *tresource.gvk)
 	}
 	// Generate
-	if client != nil {
-		engine.Generate(client, *policy, tresource.rawResource, *tresource.gvk)
-	}
 	// transform the patched Resource into resource Info
-	ri, err := extractResourceRaw(patchedResource)
+	ri, err := extractResourceRaw(rawResource)
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, err
 	}
 	// return the results
-	return ri, mResult, vResult, nil
+	return ri, policyInfo, nil
 }
 
 func NewTest(ap string, t *testing.T, tc *testCase) (*test, error) {

pkg/testrunner/testcase.go

@@ -32,22 +32,30 @@ type tInput struct {
 }
 
 type tExpected struct {
+	Passes     string       `yaml:"passes"`
 	Mutation   *tMutation   `yaml:"mutation,omitempty"`
 	Validation *tValidation `yaml:"validation,omitempty"`
 	Generation *tGeneration `yaml:"generation,omitempty"`
 }
 
 type tMutation struct {
-	Patched_Resource string `yaml:"patched_resource,omitempty"`
-	tResult
+	PatchedResource string   `yaml:"patched_resource,omitempty"`
+	Rules           []tRules `yaml:"rules"`
 }
 
 type tValidation struct {
-	tResult
+	Rules []tRules `yaml:"rules"`
 }
 
 type tGeneration struct {
 	Resources []string `yaml:"resources"`
+	Rules     []tRules `yaml:"rules"`
+}
+
+type tRules struct {
+	Name     string   `yaml:"name"`
+	Type     string   `yaml:"type"`
+	Messages []string `yaml:"messages"`
 }
 
 type tResult struct {
@@ -73,7 +81,7 @@ func (tc *testCase) loadPatchedResource(ap string) (*resourceInfo, error) {
 	if tc.Expected.Mutation == nil {
 		return nil, nil
 	}
-	rs, err := loadResources(ap, tc.Expected.Mutation.Patched_Resource)
+	rs, err := loadResources(ap, tc.Expected.Mutation.PatchedResource)
 	if err != nil {
 		return nil, err
 	}

pkg/testrunner/testrunner_test.go

@@ -2,6 +2,14 @@ package testrunner
 
 import "testing"
 
-func TestExamples(t *testing.T) {
-	runner(t, "/test/scenarios")
+func TestMutate(t *testing.T) {
+	runner(t, "/test/scenarios/mutate")
+}
+
+func TestCLI(t *testing.T) {
+	runner(t, "/test/scenarios/cli")
+}
+
+func TestGenerate(t *testing.T) {
+	runner(t, "/test/scenarios/generate")
 }

test/output/ghost.yaml

@@ -27,6 +27,7 @@ spec:
       containers:
       - name: ghost
         image: nginx:latest
+        imagePullPolicy: Always
         ports:
         - containerPort: 8080
           protocol: TCP

test/output/op_overlay_nginx.yaml

@@ -22,10 +22,10 @@ spec:
         ports:
         - containerPort: 80
         resources: {}
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
       - name: ghost
         image: ghost:latest
         resources: {}
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
   strategy: {}
 status: {}

test/scenarios/cli.yaml

@@ -1,20 +0,0 @@
-# file path relative to project root
-input:
-  policy: examples/cli/policy_deployment.yaml
-  resource: examples/cli/nginx.yaml
-expected:
-  mutation:
-    patched_resource: test/output/nginx.yaml
-    reason: Success
-  validation:
-    reason: Success
----
-input:
-  policy: examples/cli/policy_deployment.yaml
-  resource: examples/cli/ghost.yaml
-expected:
-  mutation:
-    patched_resource: test/output/ghost.yaml
-    reason: Success
-  validation:
-    reason: Success

test/scenarios/cli/cli.yaml

@@ -0,0 +1,38 @@
+# file path relative to project root
+input:
+  policy: examples/cli/policy_deployment.yaml
+  resource: examples/cli/nginx.yaml
+expected:
+  passes: true
+  mutation:
+    patched_resource: test/output/nginx.yaml
+    rules:
+      - name: add-label
+        type: Mutation
+        messages:
+          - "Rule add-label: Patches succesfully applied."
+  validation:
+    rules:
+      - name: check-image
+        type : Validation
+        messages:
+          - "Rule check-image: Validation succesfully."
+---
+input:
+  policy: examples/cli/policy_deployment.yaml
+  resource: examples/cli/ghost.yaml
+expected:
+  passes: true
+  mutation:
+    patched_resource: test/output/ghost.yaml
+    rules:
+      - name: add-label
+        type: Mutation
+        messages:
+          - "Rule add-label: Patches succesfully applied."
+  validation:
+    rules:
+      - name: check-image
+        type : Validation
+        messages:
+          - "Rule check-image: Validation succesfully."

test/scenarios/generate.yaml

@@ -1,30 +0,0 @@
-# file path relative to project root
-input:
-  policy: examples/generate/policy_basic.yaml
-  resource: examples/generate/namespace.yaml
-  load_resources:
-    - examples/generate/configMap_default.yaml
-expected:
-  generation:
-    resources:
-      - test/output/cm_default_config.yaml
-      - test/output/sc_mongo_cred.yaml
----
-input:
-  policy: examples/generate/policy_generate.yaml
-  resource: examples/generate/namespace.yaml
-  load_resources:
-    - examples/generate/configMap.yaml
-expected:
-  generation:
-    resources:
-      - test/output/cm_copied_cm.yaml
-      - test/output/cm_zk-kafka-address.yaml
----
-input:
-  policy: examples/generate/policy_networkPolicy.yaml
-  resource: examples/generate/namespace.yaml
-expected:
-  generation:
-    resources:
-      - test/output/np_deny-all-traffic.yaml

test/scenarios/generate/generate.yaml

@@ -0,0 +1,58 @@
+# file path relative to project root
+input:
+  policy: examples/generate/policy_basic.yaml
+  resource: examples/generate/namespace.yaml
+  load_resources:
+    - examples/generate/configMap_default.yaml
+expected:
+  passes: true
+  generation:
+    resources:
+      - test/output/cm_default_config.yaml
+      - test/output/sc_mongo_cred.yaml
+    rules:
+      - name: "Basic config generator for all namespaces"
+        type: Generation
+        messages: 
+          - "Rule Basic config generator for all namespaces: Generation succesfully."
+      - name: "Basic clone config generator for all namespaces"
+        type: Generation
+        messages: 
+          - "Rule Basic clone config generator for all namespaces: Generation succesfully."
+
+---
+input:
+  policy: examples/generate/policy_generate.yaml
+  resource: examples/generate/namespace.yaml
+  load_resources:
+    - examples/generate/configMap.yaml
+expected:
+  passes: true
+  generation:
+    resources:
+      - test/output/cm_copied_cm.yaml
+      - test/output/cm_zk-kafka-address.yaml
+    rules:
+      - name: "copy-comfigmap"
+        type: Generation
+        messages: 
+          - "Rule copy-comfigmap: Generation succesfully."
+      - name: "zk-kafka-address"
+        type: Generation
+        messages: 
+          - "Rule zk-kafka-address: Generation succesfully."
+
+---
+input:
+  policy: examples/generate/policy_networkPolicy.yaml
+  resource: examples/generate/namespace.yaml
+expected:
+  passes: true
+  generation:
+    resources:
+      - test/output/np_deny-all-traffic.yaml
+    rules:
+      - name: deny-all-traffic
+        type: Generation
+        messages:
+          - "Rule deny-all-traffic: Generation succesfully."

test/scenarios/mutate/overlay.yaml

@@ -3,6 +3,11 @@ input:
   policy: examples/mutate/overlay/policy_imagePullPolicy.yaml
   resource: examples/mutate/overlay/nginx.yaml
 expected:
+  passes: true
   mutation:
     patched_resource: test/output/op_overlay_nginx.yaml
-    reason: Success
+    rules:
+      - name: set-image-pull-policy
+        type: Mutation
+        messages:
+          - "Rule set-image-pull-policy: Overlay succesfully applied."

test/scenarios/mutate/patches.yaml

@@ -3,6 +3,11 @@ input:
   policy: examples/mutate/patches/policy_endpoints.yaml
   resource: examples/mutate/patches/endpoints.yaml
 expected:
+  passes: true
   mutation:
     patched_resource: test/output/op_patches_endpoints.yaml
-    reason: Success
+    rules:
+      - name: pEP
+        type: Mutation
+        messages:
+          - "Rule pEP: Patches succesfully applied."