From 725a94cc37874583c92bb80f2ea6cec3bd09b3b2 Mon Sep 17 00:00:00 2001
From: shivkumar dudhani <shivkumar@nirmata.com>
Date: Mon, 1 Jul 2019 12:16:12 -0700
Subject: [PATCH] refactor testrunner framework
---
examples/cli/ghost.yaml | 1 +
examples/generate/policy_basic.yaml | 2 +-
pkg/engine/patches.go | 6 +-
pkg/testrunner/test.go | 140 +++++++++++++++++++-------
pkg/testrunner/testcase.go | 16 ++-
pkg/testrunner/testrunner_test.go | 12 ++-
test/output/ghost.yaml | 1 +
test/output/op_overlay_nginx.yaml | 4 +-
test/scenarios/cli.yaml | 20 ----
test/scenarios/cli/cli.yaml | 38 +++++++
test/scenarios/generate.yaml | 30 ------
test/scenarios/generate/generate.yaml | 58 +++++++++++
test/scenarios/mutate/overlay.yaml | 7 +-
test/scenarios/mutate/patches.yaml | 7 +-
14 files changed, 242 insertions(+), 100 deletions(-)
delete mode 100644 test/scenarios/cli.yaml
create mode 100644 test/scenarios/cli/cli.yaml
delete mode 100644 test/scenarios/generate.yaml
create mode 100644 test/scenarios/generate/generate.yaml
diff --git a/examples/cli/ghost.yaml b/examples/cli/ghost.yaml
index 47ef19feb7..eeca6d0579 100644
--- a/examples/cli/ghost.yaml
+++ b/examples/cli/ghost.yaml
@@ -29,6 +29,7 @@ spec:
containers:
- name: "ghost"
image: "nginx:latest"
+ imagePullPolicy: Always
ports:
- containerPort: 8080
protocol: "TCP"
diff --git a/examples/generate/policy_basic.yaml b/examples/generate/policy_basic.yaml
index 79f92ef081..8e8c7e2286 100644
--- a/examples/generate/policy_basic.yaml
+++ b/examples/generate/policy_basic.yaml
@@ -4,7 +4,7 @@ metadata :
name : basic-policy
spec :
rules:
- - name: "Basic config generator for all namespaces"
+ - name: "Basic clone config generator for all namespaces"
resource:
kinds:
- Namespace
diff --git a/pkg/engine/patches.go b/pkg/engine/patches.go
index 578023f2c0..729476c668 100644
--- a/pkg/engine/patches.go
+++ b/pkg/engine/patches.go
@@ -16,10 +16,14 @@ type PatchBytes []byte
// ProcessPatches Returns array from separate patches that can be applied to the document
// Returns error ONLY in case when creation of resource should be denied.
func ProcessPatches(rule kubepolicy.Rule, resource []byte) (allPatches []PatchBytes, errs []error) {
- if len(resource) == 0 || rule.Mutation == nil {
+ if len(resource) == 0 {
errs = append(errs, errors.New("Source document for patching is empty"))
return nil, errs
}
+ if rule.Mutation == nil {
+ errs = append(errs, errors.New("No Mutation rules defined"))
+ return nil, errs
+ }
patchedDocument := resource
for _, patch := range rule.Mutation.Patches {
patchRaw, err := json.Marshal(patch)
diff --git a/pkg/testrunner/test.go b/pkg/testrunner/test.go
index f475f2e579..5230444e5b 100644
--- a/pkg/testrunner/test.go
+++ b/pkg/testrunner/test.go
@@ -1,7 +1,7 @@
package testrunner
import (
- "fmt"
+ "strconv"
"testing"
ospath "path"
@@ -10,7 +10,7 @@ import (
pt "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1"
client "github.com/nirmata/kyverno/pkg/dclient"
"github.com/nirmata/kyverno/pkg/engine"
- "github.com/nirmata/kyverno/pkg/result"
+ "github.com/nirmata/kyverno/pkg/info"
kscheme "k8s.io/client-go/kubernetes/scheme"
)
@@ -48,56 +48,102 @@ func (t *test) run() {
}
// apply the policy engine
- pr, mResult, vResult, err := t.applyPolicy(t.policy, t.tResource, client)
+ pr, policyInfo, err := t.applyPolicy(t.policy, t.tResource, client)
if err != nil {
t.t.Error(err)
return
}
// Expected Result
- t.checkMutationResult(pr, mResult)
- t.checkValidationResult(vResult)
- t.checkGenerationResult(client)
+ // Test succesfuly ?
+ t.overAllPass(policyInfo.IsSuccessful(), t.testCase.Expected.Passes)
+ t.checkMutationResult(pr, policyInfo)
+ t.checkValidationResult(policyInfo)
+ t.checkGenerationResult(client, policyInfo)
}
-func (t *test) checkMutationResult(pr *resourceInfo, result result.Result) {
+func (t *test) checkMutationResult(pr *resourceInfo, policyInfo *info.PolicyInfo) {
if t.testCase.Expected.Mutation == nil {
glog.Info("No Mutation check defined")
return
}
// patched resource
if !compareResource(pr, t.patchedResource) {
- fmt.Printf("Expected Resource %s \n", string(t.patchedResource.rawResource))
- fmt.Printf("Patched Resource %s \n", string(pr.rawResource))
glog.Warningf("Expected resource %s ", string(pr.rawResource))
t.t.Error("Patched resources not as expected")
}
- // reason
- reason := t.testCase.Expected.Mutation.Reason
- if len(reason) > 0 && result.GetReason().String() != reason {
- t.t.Error("Reason not matching")
+
+ // check if rules match
+ t.compareRules(policyInfo.Rules, t.testCase.Expected.Mutation.Rules)
+}
+
+func (t *test) overAllPass(result bool, expected string) {
+ b, err := strconv.ParseBool(expected)
+ if err != nil {
+ t.t.Error(err)
+ }
+ if result != b {
+ t.t.Errorf("Expected value %v and actual value %v dont match", expected, result)
}
}
-func (t *test) checkValidationResult(result result.Result) {
+func (t *test) compareRules(ruleInfos []*info.RuleInfo, rules []tRules) {
+ // Compare the rules specified in the expected against the actual rule info returned by the apply policy
+ for _, eRule := range rules {
+ // Look-up the rule from the policy info
+ rule := lookUpRule(eRule.Name, ruleInfos)
+ if rule == nil {
+ t.t.Errorf("Rule with name %s not found", eRule.Name)
+ continue
+ }
+ // get the corresponding rule
+ if rule.Name != eRule.Name {
+ t.t.Errorf("Rule Name not matching!. expected %s , actual %s", eRule.Name, rule.Name)
+ }
+ if rule.RuleType.String() != eRule.Type {
+ t.t.Errorf("Rule type mismatch!. expected %s, actual %s", eRule.Type, rule.RuleType.String())
+ }
+ if len(eRule.Messages) != len(rule.Msgs) {
+ t.t.Errorf("Number of rule messages not same. expected %d, actual %d", len(eRule.Messages), len(rule.Msgs))
+ }
+ for i, msg := range eRule.Messages {
+ if msg != rule.Msgs[i] {
+ t.t.Errorf("Messges dont match!. expected %s, actual %s", msg, rule.Msgs[i])
+ }
+ }
+ }
+}
+
+func lookUpRule(name string, ruleInfos []*info.RuleInfo) *info.RuleInfo {
+ for _, r := range ruleInfos {
+ if r.Name == name {
+ return r
+ }
+ }
+ return nil
+}
+
+func (t *test) checkValidationResult(policyInfo *info.PolicyInfo) {
if t.testCase.Expected.Validation == nil {
glog.Info("No Validation check defined")
return
}
- // reason
- reason := t.testCase.Expected.Validation.Reason
- if len(reason) > 0 && result.GetReason().String() != reason {
- t.t.Error("Reason not matching")
- }
+
+ // check if rules match
+ t.compareRules(policyInfo.Rules, t.testCase.Expected.Validation.Rules)
}
-func (t *test) checkGenerationResult(client *client.Client) {
+func (t *test) checkGenerationResult(client *client.Client, policyInfo *info.PolicyInfo) {
if t.testCase.Expected.Generation == nil {
glog.Info("No Generate check defined")
return
}
if client == nil {
- glog.Info("client needs to be configured")
+ t.t.Error("client needs to be configured")
}
+
+ // check if rules match
+ t.compareRules(policyInfo.Rules, t.testCase.Expected.Generation.Rules)
+
// check if the expected resources are generated
for _, r := range t.genResources {
n := ParseNameFromObject(r.rawResource)
@@ -113,33 +159,51 @@ func (t *test) checkGenerationResult(client *client.Client) {
func (t *test) applyPolicy(policy *pt.Policy,
tresource *resourceInfo,
- client *client.Client) (*resourceInfo, result.Result, result.Result, error) {
+ client *client.Client) (*resourceInfo, *info.PolicyInfo, error) {
// apply policy on the trigger resource
// Mutate
- var vResult result.Result
- var patchedResource []byte
- mPatches, mResult := engine.Mutate(*policy, tresource.rawResource, *tresource.gvk)
+ var err error
+ rawResource := tresource.rawResource
+ rname := engine.ParseNameFromObject(rawResource)
+ rns := engine.ParseNamespaceFromObject(rawResource)
+ rkind := engine.ParseKindFromObject(rawResource)
+ policyInfo := info.NewPolicyInfo(policy.Name,
+ rkind,
+ rname,
+ rns)
+ // Apply Mutation Rules
+ patches, ruleInfos := engine.Mutate(*policy, rawResource, *tresource.gvk)
+ policyInfo.AddRuleInfos(ruleInfos)
// TODO: only validate if there are no errors in mutate, why?
- err := mResult.ToError()
- if err == nil && len(mPatches) != 0 {
- patchedResource, err = engine.ApplyPatches(tresource.rawResource, mPatches)
- if err != nil {
- return nil, nil, nil, err
+ if policyInfo.IsSuccessful() {
+ if len(patches) != 0 {
+ rawResource, err = engine.ApplyPatches(rawResource, patches)
+ if err != nil {
+ return nil, nil, err
+ }
+ }
+ }
+ // Validate
+ ruleInfos, err = engine.Validate(*policy, rawResource, *tresource.gvk)
+ policyInfo.AddRuleInfos(ruleInfos)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ if rkind == "Namespace" {
+ if client != nil {
+ ruleInfos := engine.Generate(client, *policy, rawResource, *tresource.gvk, false)
+ policyInfo.AddRuleInfos(ruleInfos)
}
- // Validate
- vResult = engine.Validate(*policy, patchedResource, *tresource.gvk)
}
// Generate
- if client != nil {
- engine.Generate(client, *policy, tresource.rawResource, *tresource.gvk)
- }
// transform the patched Resource into resource Info
- ri, err := extractResourceRaw(patchedResource)
+ ri, err := extractResourceRaw(rawResource)
if err != nil {
- return nil, nil, nil, err
+ return nil, nil, err
}
// return the results
- return ri, mResult, vResult, nil
+ return ri, policyInfo, nil
}
func NewTest(ap string, t *testing.T, tc *testCase) (*test, error) {
diff --git a/pkg/testrunner/testcase.go b/pkg/testrunner/testcase.go
index 1bbdf70291..80363c7b1e 100644
--- a/pkg/testrunner/testcase.go
+++ b/pkg/testrunner/testcase.go
@@ -32,22 +32,30 @@ type tInput struct {
}
type tExpected struct {
+ Passes string `yaml:"passes"`
Mutation *tMutation `yaml:"mutation,omitempty"`
Validation *tValidation `yaml:"validation,omitempty"`
Generation *tGeneration `yaml:"generation,omitempty"`
}
type tMutation struct {
- Patched_Resource string `yaml:"patched_resource,omitempty"`
- tResult
+ PatchedResource string `yaml:"patched_resource,omitempty"`
+ Rules []tRules `yaml:"rules"`
}
type tValidation struct {
- tResult
+ Rules []tRules `yaml:"rules"`
}
type tGeneration struct {
Resources []string `yaml:"resources"`
+ Rules []tRules `yaml:"rules"`
+}
+
+type tRules struct {
+ Name string `yaml:"name"`
+ Type string `yaml:"type"`
+ Messages []string `yaml:"messages"`
}
type tResult struct {
@@ -73,7 +81,7 @@ func (tc *testCase) loadPatchedResource(ap string) (*resourceInfo, error) {
if tc.Expected.Mutation == nil {
return nil, nil
}
- rs, err := loadResources(ap, tc.Expected.Mutation.Patched_Resource)
+ rs, err := loadResources(ap, tc.Expected.Mutation.PatchedResource)
if err != nil {
return nil, err
}
diff --git a/pkg/testrunner/testrunner_test.go b/pkg/testrunner/testrunner_test.go
index 50d949effd..c77e875da9 100644
--- a/pkg/testrunner/testrunner_test.go
+++ b/pkg/testrunner/testrunner_test.go
@@ -2,6 +2,14 @@ package testrunner
import "testing"
-func TestExamples(t *testing.T) {
- runner(t, "/test/scenarios")
+func TestMutate(t *testing.T) {
+ runner(t, "/test/scenarios/mutate")
+}
+
+func TestCLI(t *testing.T) {
+ runner(t, "/test/scenarios/cli")
+}
+
+func TestGenerate(t *testing.T) {
+ runner(t, "/test/scenarios/generate")
}
diff --git a/test/output/ghost.yaml b/test/output/ghost.yaml
index 751db9ec4c..cda52a8cc2 100644
--- a/test/output/ghost.yaml
+++ b/test/output/ghost.yaml
@@ -27,6 +27,7 @@ spec:
containers:
- name: ghost
image: nginx:latest
+ imagePullPolicy: Always
ports:
- containerPort: 8080
protocol: TCP
diff --git a/test/output/op_overlay_nginx.yaml b/test/output/op_overlay_nginx.yaml
index d59519c5a7..a45c4fdc2e 100644
--- a/test/output/op_overlay_nginx.yaml
+++ b/test/output/op_overlay_nginx.yaml
@@ -22,10 +22,10 @@ spec:
ports:
- containerPort: 80
resources: {}
- imagePullPolicy: Always
+ imagePullPolicy: IfNotPresent
- name: ghost
image: ghost:latest
resources: {}
- imagePullPolicy: Always
+ imagePullPolicy: IfNotPresent
strategy: {}
status: {}
diff --git a/test/scenarios/cli.yaml b/test/scenarios/cli.yaml
deleted file mode 100644
index 1e9496b983..0000000000
--- a/test/scenarios/cli.yaml
+++ /dev/null
@@ -1,20 +0,0 @@
-# file path relative to project root
-input:
- policy: examples/cli/policy_deployment.yaml
- resource: examples/cli/nginx.yaml
-expected:
- mutation:
- patched_resource: test/output/nginx.yaml
- reason: Success
- validation:
- reason: Success
----
-input:
- policy: examples/cli/policy_deployment.yaml
- resource: examples/cli/ghost.yaml
-expected:
- mutation:
- patched_resource: test/output/ghost.yaml
- reason: Success
- validation:
- reason: Success
\ No newline at end of file
diff --git a/test/scenarios/cli/cli.yaml b/test/scenarios/cli/cli.yaml
new file mode 100644
index 0000000000..8e1b2c2d96
--- /dev/null
+++ b/test/scenarios/cli/cli.yaml
@@ -0,0 +1,38 @@
+# file path relative to project root
+input:
+ policy: examples/cli/policy_deployment.yaml
+ resource: examples/cli/nginx.yaml
+expected:
+ passes: true
+ mutation:
+ patched_resource: test/output/nginx.yaml
+ rules:
+ - name: add-label
+ type: Mutation
+ messages:
+ - "Rule add-label: Patches succesfully applied."
+ validation:
+ rules:
+ - name: check-image
+ type : Validation
+ messages:
+ - "Rule check-image: Validation succesfully."
+---
+input:
+ policy: examples/cli/policy_deployment.yaml
+ resource: examples/cli/ghost.yaml
+expected:
+ passes: true
+ mutation:
+ patched_resource: test/output/ghost.yaml
+ rules:
+ - name: add-label
+ type: Mutation
+ messages:
+ - "Rule add-label: Patches succesfully applied."
+ validation:
+ rules:
+ - name: check-image
+ type : Validation
+ messages:
+ - "Rule check-image: Validation succesfully."
diff --git a/test/scenarios/generate.yaml b/test/scenarios/generate.yaml
deleted file mode 100644
index b305067a04..0000000000
--- a/test/scenarios/generate.yaml
+++ /dev/null
@@ -1,30 +0,0 @@
-# file path relative to project root
-input:
- policy: examples/generate/policy_basic.yaml
- resource: examples/generate/namespace.yaml
- load_resources:
- - examples/generate/configMap_default.yaml
-expected:
- generation:
- resources:
- - test/output/cm_default_config.yaml
- - test/output/sc_mongo_cred.yaml
----
-input:
- policy: examples/generate/policy_generate.yaml
- resource: examples/generate/namespace.yaml
- load_resources:
- - examples/generate/configMap.yaml
-expected:
- generation:
- resources:
- - test/output/cm_copied_cm.yaml
- - test/output/cm_zk-kafka-address.yaml
----
-input:
- policy: examples/generate/policy_networkPolicy.yaml
- resource: examples/generate/namespace.yaml
-expected:
- generation:
- resources:
- - test/output/np_deny-all-traffic.yaml
\ No newline at end of file
diff --git a/test/scenarios/generate/generate.yaml b/test/scenarios/generate/generate.yaml
new file mode 100644
index 0000000000..572efe2ad1
--- /dev/null
+++ b/test/scenarios/generate/generate.yaml
@@ -0,0 +1,58 @@
+# file path relative to project root
+input:
+ policy: examples/generate/policy_basic.yaml
+ resource: examples/generate/namespace.yaml
+ load_resources:
+ - examples/generate/configMap_default.yaml
+expected:
+ passes: true
+ generation:
+ resources:
+ - test/output/cm_default_config.yaml
+ - test/output/sc_mongo_cred.yaml
+ rules:
+ - name: "Basic config generator for all namespaces"
+ type: Generation
+ messages:
+ - "Rule Basic config generator for all namespaces: Generation succesfully."
+ - name: "Basic clone config generator for all namespaces"
+ type: Generation
+ messages:
+ - "Rule Basic clone config generator for all namespaces: Generation succesfully."
+
+---
+input:
+ policy: examples/generate/policy_generate.yaml
+ resource: examples/generate/namespace.yaml
+ load_resources:
+ - examples/generate/configMap.yaml
+expected:
+ passes: true
+ generation:
+ resources:
+ - test/output/cm_copied_cm.yaml
+ - test/output/cm_zk-kafka-address.yaml
+ rules:
+ - name: "copy-comfigmap"
+ type: Generation
+ messages:
+ - "Rule copy-comfigmap: Generation succesfully."
+ - name: "zk-kafka-address"
+ type: Generation
+ messages:
+ - "Rule zk-kafka-address: Generation succesfully."
+
+---
+input:
+ policy: examples/generate/policy_networkPolicy.yaml
+ resource: examples/generate/namespace.yaml
+expected:
+ passes: true
+ generation:
+ resources:
+ - test/output/np_deny-all-traffic.yaml
+ rules:
+ - name: deny-all-traffic
+ type: Generation
+ messages:
+ - "Rule deny-all-traffic: Generation succesfully."
\ No newline at end of file
diff --git a/test/scenarios/mutate/overlay.yaml b/test/scenarios/mutate/overlay.yaml
index a2db4326c5..95b606bda5 100644
--- a/test/scenarios/mutate/overlay.yaml
+++ b/test/scenarios/mutate/overlay.yaml
@@ -3,6 +3,11 @@ input:
policy: examples/mutate/overlay/policy_imagePullPolicy.yaml
resource: examples/mutate/overlay/nginx.yaml
expected:
+ passes: true
mutation:
patched_resource: test/output/op_overlay_nginx.yaml
- reason: Success
\ No newline at end of file
+ rules:
+ - name: set-image-pull-policy
+ type: Mutation
+ messages:
+ - "Rule set-image-pull-policy: Overlay succesfully applied."
\ No newline at end of file
diff --git a/test/scenarios/mutate/patches.yaml b/test/scenarios/mutate/patches.yaml
index 70de56af2f..6b5201c382 100644
--- a/test/scenarios/mutate/patches.yaml
+++ b/test/scenarios/mutate/patches.yaml
@@ -3,6 +3,11 @@ input:
policy: examples/mutate/patches/policy_endpoints.yaml
resource: examples/mutate/patches/endpoints.yaml
expected:
+ passes: true
mutation:
patched_resource: test/output/op_patches_endpoints.yaml
- reason: Success
\ No newline at end of file
+ rules:
+ - name: pEP
+ type: Mutation
+ messages:
+ - "Rule pEP: Patches succesfully applied."
\ No newline at end of file