rework the framework

pkg/testrunner/test.go

@@ -0,0 +1,180 @@
+package testrunner
+
+import (
+	"fmt"
+	"testing"
+
+	ospath "path"
+
+	"github.com/golang/glog"
+	pt "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1"
+	client "github.com/nirmata/kyverno/pkg/dclient"
+	"github.com/nirmata/kyverno/pkg/engine"
+	"github.com/nirmata/kyverno/pkg/result"
+	kscheme "k8s.io/client-go/kubernetes/scheme"
+)
+
+type test struct {
+	ap       string
+	t        *testing.T
+	testCase *testCase
+	// input
+	policy        *pt.Policy
+	tResource     *resourceInfo
+	loadResources []*resourceInfo
+	// expected
+	genResources    []*resourceInfo
+	patchedResource *resourceInfo
+}
+
+func (t *test) run() {
+	var client *client.Client
+	var err error
+	//mock client is used if generate is defined
+	if t.testCase.Expected.Generation != nil {
+		// create mock client & load resources
+		client, err = createClient(t.loadResources)
+		if err != nil {
+			t.t.Errorf("Unable to create client. err %s", err)
+		}
+		// TODO: handle generate
+		// assuming its namespaces creation
+		decode := kscheme.Codecs.UniversalDeserializer().Decode
+		obj, _, err := decode([]byte(t.tResource.rawResource), nil, nil)
+		_, err = client.CreateResource(getResourceFromKind(t.tResource.gvk.Kind), "", obj)
+		if err != nil {
+			t.t.Errorf("error while creating namespace %s", err)
+		}
+
+	}
+	// apply the policy engine
+	pr, mResult, vResult, err := t.applyPolicy(t.policy, t.tResource, client)
+	if err != nil {
+		t.t.Error(err)
+		return
+	}
+	// Expected Result
+	t.checkMutationResult(pr, mResult)
+	t.checkValidationResult(vResult)
+	t.checkGenerationResult(client)
+}
+
+func (t *test) checkMutationResult(pr *resourceInfo, result result.Result) {
+	if t.testCase.Expected.Mutation == nil {
+		glog.Info("No Mutation check defined")
+		return
+	}
+	// patched resource
+	if !compareResource(pr, t.patchedResource) {
+		fmt.Printf("Expected Resource %s \n", string(t.patchedResource.rawResource))
+		fmt.Printf("Patched Resource %s \n", string(pr.rawResource))
+		glog.Warningf("Expected resource %s ", string(pr.rawResource))
+		t.t.Error("Patched resources not as expected")
+	}
+	// reason
+	reason := t.testCase.Expected.Mutation.Reason
+	if len(reason) > 0 && result.GetReason().String() != reason {
+		t.t.Error("Reason not matching")
+	}
+}
+
+func (t *test) checkValidationResult(result result.Result) {
+	if t.testCase.Expected.Validation == nil {
+		glog.Info("No Validation check defined")
+		return
+	}
+	// reason
+	reason := t.testCase.Expected.Validation.Reason
+	if len(reason) > 0 && result.GetReason().String() != reason {
+		t.t.Error("Reason not matching")
+	}
+}
+
+func (t *test) checkGenerationResult(client *client.Client) {
+	if t.testCase.Expected.Generation == nil {
+		glog.Info("No Generate check defined")
+		return
+	}
+	if client == nil {
+		glog.Info("client needs to be configured")
+	}
+	// check if the expected resources are generated
+	for _, r := range t.genResources {
+		n := ParseNameFromObject(r.rawResource)
+		ns := ParseNamespaceFromObject(r.rawResource)
+		_, err := client.GetResource(getResourceFromKind(r.gvk.Kind), ns, n)
+		if err != nil {
+			t.t.Errorf("Resource %s/%s of kinf %s not found", ns, n, r.gvk.Kind)
+		}
+		// compare if the resources are same
+		//TODO: comapre []bytes vs unstrcutured resource
+	}
+}
+
+func (t *test) applyPolicy(policy *pt.Policy,
+	tresource *resourceInfo,
+	client *client.Client) (*resourceInfo, result.Result, result.Result, error) {
+	// apply policy on the trigger resource
+	// Mutate
+	var vResult result.Result
+	var patchedResource []byte
+	mPatches, mResult := engine.Mutate(*policy, tresource.rawResource, *tresource.gvk)
+	// TODO: only validate if there are no errors in mutate, why?
+	err := mResult.ToError()
+	if err == nil && len(mPatches) != 0 {
+		patchedResource, err = engine.ApplyPatches(tresource.rawResource, mPatches)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		// Validate
+		vResult = engine.Validate(*policy, patchedResource, *tresource.gvk)
+	}
+	// Generate
+	if client != nil {
+		engine.Generate(client, *policy, tresource.rawResource, *tresource.gvk)
+	}
+	// transform the patched Resource into resource Info
+	ri, err := extractResourceRaw(patchedResource)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	// return the results
+	return ri, mResult, vResult, nil
+}
+
+func NewTest(ap string, t *testing.T, tc *testCase) (*test, error) {
+	//---INPUT---
+	p, err := tc.loadPolicy(ospath.Join(ap, tc.Input.Policy))
+	if err != nil {
+		return nil, err
+	}
+	r, err := tc.loadTriggerResource(ap)
+	if err != nil {
+		return nil, err
+	}
+
+	lr, err := tc.loadPreloadedResources(ap)
+	if err != nil {
+		return nil, err
+	}
+
+	//---EXPECTED---
+	pr, err := tc.loadPatchedResource(ap)
+	if err != nil {
+		return nil, err
+	}
+	gr, err := tc.loadGeneratedResources(ap)
+	if err != nil {
+		return nil, err
+	}
+	return &test{
+		ap:              ap,
+		t:               t,
+		testCase:        tc,
+		policy:          p,
+		tResource:       r,
+		loadResources:   lr,
+		genResources:    gr,
+		patchedResource: pr,
+	}, nil
+}

pkg/testrunner/testcase.go

@@ -0,0 +1,178 @@
+package testrunner
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	ospath "path"
+
+	"github.com/golang/glog"
+	pt "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	yaml "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/kubernetes/scheme"
+)
+
+//testCase defines the input and the expected result
+// it stores the path to the files that are to be loaded
+// for references
+type testCase struct {
+	Input    *tInput    `yaml:"input"`
+	Expected *tExpected `yaml:"expected"`
+}
+
+// load resources store the resources that are pre-requisite
+// for the test case and are pre-loaded in the client before
+/// test case in evaluated
+type tInput struct {
+	Policy        string   `yaml:"policy"`
+	Resource      string   `yaml:"resource"`
+	LoadResources []string `yaml:"load_resources,omitempty"`
+}
+
+type tExpected struct {
+	Mutation   *tMutation   `yaml:"mutation,omitempty"`
+	Validation *tValidation `yaml:"validation,omitempty"`
+	Generation *tGeneration `yaml:"generation,omitempty"`
+}
+
+type tMutation struct {
+	Patched_Resource string `yaml:"patched_resource,omitempty"`
+	tResult
+}
+
+type tValidation struct {
+	tResult
+}
+
+type tGeneration struct {
+	Resources []string `yaml:"resources"`
+}
+
+type tResult struct {
+	Reason string `yaml:"reason, omitempty"`
+}
+
+func (tc *testCase) policyEngineTest() {
+
+}
+func (tc *testCase) loadPreloadedResources(ap string) ([]*resourceInfo, error) {
+	return loadResources(ap, tc.Input.LoadResources...)
+	//	return loadResources(ap, tc.Input.LoadResources...)
+}
+
+func (tc *testCase) loadGeneratedResources(ap string) ([]*resourceInfo, error) {
+	if tc.Expected.Generation == nil {
+		return nil, nil
+	}
+	return loadResources(ap, tc.Expected.Generation.Resources...)
+}
+
+func (tc *testCase) loadPatchedResource(ap string) (*resourceInfo, error) {
+	if tc.Expected.Mutation == nil {
+		return nil, nil
+	}
+	rs, err := loadResources(ap, tc.Expected.Mutation.Patched_Resource)
+	if err != nil {
+		return nil, err
+	}
+	if len(rs) != 1 {
+		glog.Warning("expects single resource mutation but multiple defined, will use first one")
+	}
+	return rs[0], nil
+
+}
+func (tc *testCase) loadResources(files []string) ([]*resourceInfo, error) {
+	lr := []*resourceInfo{}
+	for _, r := range files {
+		rs, err := loadResources(r)
+		if err != nil {
+			// return as test case will be invalid if a resource cannot be loaded
+			return nil, err
+		}
+		lr = append(lr, rs...)
+	}
+	return lr, nil
+}
+
+func (tc *testCase) loadTriggerResource(ap string) (*resourceInfo, error) {
+	rs, err := loadResources(ap, tc.Input.Resource)
+	if err != nil {
+		return nil, err
+	}
+	if len(rs) != 1 {
+		glog.Warning("expects single resource trigger but multiple defined, will use first one")
+	}
+	return rs[0], nil
+
+}
+
+// Loads a single policy
+func (tc *testCase) loadPolicy(file string) (*pt.Policy, error) {
+	p := &pt.Policy{}
+	data, err := LoadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	pBytes, err := yaml.ToJSON(data)
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(pBytes, p); err != nil {
+		return nil, err
+	}
+	if p.TypeMeta.Kind != "Policy" {
+		return nil, fmt.Errorf("failed to parse policy")
+	}
+	return p, nil
+}
+
+// loads multiple resources
+func loadResources(ap string, args ...string) ([]*resourceInfo, error) {
+	ris := []*resourceInfo{}
+	for _, file := range args {
+		data, err := LoadFile(ospath.Join(ap, file))
+		if err != nil {
+			return nil, err
+		}
+		dd := bytes.Split(data, []byte(defaultYamlSeparator))
+		// resources seperated by yaml seperator
+		for _, d := range dd {
+			ri, err := extractResourceRaw(d)
+			if err != nil {
+				glog.Errorf("unable to load resource. err: %s ", err)
+				continue
+			}
+			ris = append(ris, ri)
+		}
+	}
+	return ris, nil
+}
+
+func extractResourceRaw(d []byte) (*resourceInfo, error) {
+	// decode := scheme.Codecs.UniversalDeserializer().Decode
+	// obj, gvk, err := decode(d, nil, nil)
+	// if err != nil {
+	// 	return nil, err
+	// }
+	obj, gvk, err := extractResourceUnMarshalled(d)
+	// runtime.object to JSON
+	raw, err := json.Marshal(obj)
+	if err != nil {
+		return nil, err
+	}
+	return &resourceInfo{rawResource: raw,
+		gvk: gvk}, nil
+}
+
+func extractResourceUnMarshalled(d []byte) (runtime.Object, *metav1.GroupVersionKind, error) {
+	decode := scheme.Codecs.UniversalDeserializer().Decode
+	obj, gvk, err := decode(d, nil, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	return obj, &metav1.GroupVersionKind{Group: gvk.Group,
+		Version: gvk.Version,
+		Kind:    gvk.Kind}, nil
+}

pkg/testrunner/testrunner.go

@@ -0,0 +1,98 @@
+package testrunner
+
+import (
+	"bytes"
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+
+	"os"
+	ospath "path"
+
+	"github.com/golang/glog"
+	"gopkg.in/yaml.v2"
+)
+
+func runner(t *testing.T, relpath string) {
+	gp := os.Getenv("GOPATH")
+	ap := ospath.Join(gp, projectPath)
+	// build load scenarios
+	path := ospath.Join(ap, relpath)
+	// Load the scenario files
+	scenarioFiles := getYAMLfiles(path)
+	for _, secenarioFile := range scenarioFiles {
+		sc := newScenario(t, ap, secenarioFile)
+		if err := sc.load(); err != nil {
+			t.Error(err)
+			return
+		}
+		// run test cases
+		sc.run()
+	}
+}
+
+type scenario struct {
+	ap   string
+	t    *testing.T
+	path string
+	tcs  []*testCase
+}
+
+func newScenario(t *testing.T, ap string, path string) *scenario {
+	return &scenario{
+		ap:   ap,
+		t:    t,
+		path: path,
+	}
+}
+
+func getYAMLfiles(path string) (yamls []string) {
+	fileInfo, err := ioutil.ReadDir(path)
+	if err != nil {
+		return nil
+	}
+	for _, file := range fileInfo {
+		if filepath.Ext(file.Name()) == ".yml" || filepath.Ext(file.Name()) == ".yaml" {
+			yamls = append(yamls, ospath.Join(path, file.Name()))
+		}
+	}
+	return yamls
+}
+func (sc *scenario) load() error {
+	// read file
+	data, err := LoadFile(sc.path)
+	if err != nil {
+		return err
+	}
+	tcs := []*testCase{}
+	// load test cases seperated by '---'
+	// each test case defines an input & expected result
+	dd := bytes.Split(data, []byte(defaultYamlSeparator))
+	for _, d := range dd {
+		tc := &testCase{}
+		err := yaml.Unmarshal([]byte(d), tc)
+		if err != nil {
+			glog.Warningf("Error while decoding YAML object, err: %s", err)
+			continue
+		}
+		tcs = append(tcs, tc)
+	}
+	sc.tcs = tcs
+	return nil
+}
+
+func (sc *scenario) run() {
+	if len(sc.tcs) == 0 {
+		sc.t.Error("No test cases to load")
+		return
+	}
+
+	for _, tc := range sc.tcs {
+		t, err := NewTest(sc.ap, sc.t, tc)
+		if err != nil {
+			sc.t.Error(err)
+			continue
+		}
+		t.run()
+	}
+}

pkg/testrunner/testrunner_test.go

@@ -0,0 +1,7 @@
+package testrunner
+
+import "testing"
+
+func TestExamples(t *testing.T) {
+	runner(t, "/test/scenarios")
+}

pkg/testrunner/utils.go

@@ -0,0 +1,123 @@
+package testrunner
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+
+	"github.com/golang/glog"
+	client "github.com/nirmata/kyverno/pkg/dclient"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	kscheme "k8s.io/client-go/kubernetes/scheme"
+)
+
+const (
+	defaultYamlSeparator = "---"
+	projectPath          = "src/github.com/nirmata/kyverno"
+)
+
+// LoadFile loads file in byte buffer
+func LoadFile(path string) ([]byte, error) {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return nil, err
+	}
+	return ioutil.ReadFile(path)
+}
+
+type resourceInfo struct {
+	rawResource []byte
+	gvk         *metav1.GroupVersionKind
+}
+
+func (ri resourceInfo) isSame(other resourceInfo) bool {
+	// compare gvk
+	if *ri.gvk != *other.gvk {
+		return false
+	}
+	// compare rawResource
+	return bytes.Equal(ri.rawResource, other.rawResource)
+}
+
+// compare patched resources
+func compareResource(er *resourceInfo, pr *resourceInfo) bool {
+	if !er.isSame(*pr) {
+		return false
+	}
+	return true
+}
+
+func createClient(resources []*resourceInfo) (*client.Client, error) {
+	scheme := runtime.NewScheme()
+	objects := []runtime.Object{}
+	// registered group versions
+	regResources := []schema.GroupVersionResource{}
+
+	for _, r := range resources {
+		// registered gvr
+		gv := schema.GroupVersion{Group: r.gvk.Group, Version: r.gvk.Version}
+		gvr := gv.WithResource(getResourceFromKind(r.gvk.Kind))
+		regResources = append(regResources, gvr)
+		decode := kscheme.Codecs.UniversalDeserializer().Decode
+		obj, _, err := decode([]byte(r.rawResource), nil, nil)
+		rdata, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&obj)
+		if err != nil {
+			glog.Errorf("failed to load resource. err %s", err)
+		}
+		unstr := unstructured.Unstructured{Object: rdata}
+		objects = append(objects, &unstr)
+	}
+	// Mock Client
+	c, err := client.NewMockClient(scheme, objects...)
+	if err != nil {
+		return nil, err
+	}
+	c.SetDiscovery(client.NewFakeDiscoveryClient(regResources))
+
+	return c, nil
+}
+
+var kindToResource = map[string]string{
+	"ConfigMap":     "configmaps",
+	"Endpoints":     "endpoints",
+	"Namespace":     "namespaces",
+	"Secret":        "secrets",
+	"Deployment":    "deployments",
+	"NetworkPolicy": "networkpolicies",
+}
+
+func getResourceFromKind(kind string) string {
+	if resource, ok := kindToResource[kind]; ok {
+		return resource
+	}
+	return ""
+}
+
+//ParseNameFromObject extracts resource name from JSON obj
+func ParseNameFromObject(bytes []byte) string {
+	var objectJSON map[string]interface{}
+	json.Unmarshal(bytes, &objectJSON)
+
+	meta := objectJSON["metadata"].(map[string]interface{})
+
+	if name, ok := meta["name"].(string); ok {
+		return name
+	}
+	return ""
+}
+
+// ParseNamespaceFromObject extracts the namespace from the JSON obj
+func ParseNamespaceFromObject(bytes []byte) string {
+	var objectJSON map[string]interface{}
+	json.Unmarshal(bytes, &objectJSON)
+
+	meta := objectJSON["metadata"].(map[string]interface{})
+
+	if namespace, ok := meta["namespace"].(string); ok {
+		return namespace
+	}
+	return ""
+}

pkg/testutils/testbundle.go

@@ -1,294 +0,0 @@
-package testutils
-
-import (
-	"bytes"
-	"fmt"
-	"os"
-	ospath "path"
-	"testing"
-
-	"github.com/golang/glog"
-	policytypes "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1"
-	dclient "github.com/nirmata/kyverno/pkg/dclient"
-	"github.com/nirmata/kyverno/pkg/result"
-	"gopkg.in/yaml.v2"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
-	kscheme "k8s.io/client-go/kubernetes/scheme"
-
-	"k8s.io/apimachinery/pkg/runtime/schema"
-)
-
-func NewTestBundle(path string) *testBundle {
-	return &testBundle{
-		path:      path,
-		policies:  make(map[string]*policytypes.Policy),
-		resources: make(map[string]*resourceInfo),
-		output:    make(map[string]*resourceInfo),
-	}
-}
-
-func loadResources(tbPath string, rs map[string]*resourceInfo, file string) {
-	path := ospath.Join(tbPath, file)
-	_, err := os.Stat(path)
-	if os.IsNotExist(err) {
-		glog.Warningf("%s directory not present at %s", file, tbPath)
-		return
-	}
-	// Load the resources from the output folder
-	yamls := getYAMLfiles(path)
-	if len(yamls) == 0 {
-		glog.Warningf("No resource yaml found at path %s", path)
-		return
-	}
-	for _, r := range yamls {
-		resources, err := extractResource(r)
-		if err != nil {
-			glog.Errorf("unable to extract resource: %s", err)
-		}
-		mergeResources(rs, resources)
-	}
-}
-
-func (tb *testBundle) loadOutput() {
-	// check if output folder is defined
-	opath := ospath.Join(tb.path, outputFolder)
-	_, err := os.Stat(opath)
-	if os.IsNotExist(err) {
-		glog.Warningf("Output directory not present at %s", tb.path)
-		return
-	}
-	// Load the resources from the output folder
-	oYAMLs := getYAMLfiles(opath)
-	if len(oYAMLs) == 0 {
-		glog.Warningf("No resource yaml found at path %s", opath)
-		return
-	}
-
-	for _, r := range oYAMLs {
-		resources, err := extractResource(r)
-		if err != nil {
-			glog.Errorf("unable to extract resource: %s", err)
-		}
-		mergeResources(tb.output, resources)
-	}
-}
-
-func loadScenarios(tbPath string, file string) ([]*tScenario, error) {
-	// check if scenario yaml is defined
-	spath := ospath.Join(tbPath, file)
-	_, err := os.Stat(spath)
-	if os.IsNotExist(err) {
-		return nil, fmt.Errorf("Scenario file %s not defined at %s", file, tbPath)
-	}
-	ts := []*tScenario{}
-	// read the file
-	data, err := loadFile(spath)
-	if err != nil {
-		glog.Warningf("Error while loading file: %v\n", err)
-		return nil, err
-	}
-	dd := bytes.Split(data, []byte(defaultYamlSeparator))
-	for _, d := range dd {
-		s := &tScenario{}
-		err := yaml.Unmarshal([]byte(d), s)
-		if err != nil {
-			glog.Warningf("Error while decoding YAML object, err: %s", err)
-			continue
-		}
-		ts = append(ts, s)
-	}
-	return ts, nil
-}
-
-// Load test structure folder
-func (tb *testBundle) load() error {
-	// scenario file defines the mapping of resources and policies
-	scenarios, err := loadScenarios(tb.path, tScenarioFile)
-	if err != nil {
-		return err
-	}
-	tb.scenarios = scenarios
-	// check if there are any files
-	pYAMLs := getYAMLfiles(tb.path)
-	if len(pYAMLs) == 0 {
-		return fmt.Errorf("No policy yaml found at path %s", tb.path)
-	}
-	for _, p := range pYAMLs {
-		// extract policy
-		policy, err := extractPolicy(p)
-		if err != nil {
-			glog.Errorf("unable to extract policy: %s", err)
-			continue
-		}
-		tb.policies[policy.GetName()] = policy
-	}
-	// load trigger resources
-	loadResources(tb.path, tb.resources, resourcesFolder)
-	// load output resources
-	loadResources(tb.path, tb.output, outputFolder)
-
-	return nil
-}
-
-func mergeResources(rs map[string]*resourceInfo, other map[string]*resourceInfo) {
-	for k, v := range other {
-		if _, ok := rs[k]; ok {
-			glog.Infof("resource already defined %s ", k)
-			continue
-		}
-		rs[k] = v
-	}
-}
-
-type testBundle struct {
-	path      string
-	policies  map[string]*policytypes.Policy
-	resources map[string]*resourceInfo
-	output    map[string]*resourceInfo
-	scenarios []*tScenario
-}
-
-func (tb *testBundle) createClient(t *testing.T, resources []string) *dclient.Client {
-	scheme := runtime.NewScheme()
-	objects := []runtime.Object{}
-	// registered group versions
-	regResources := []schema.GroupVersionResource{}
-	for _, resource := range resources {
-		// get resources
-		r, ok := tb.resources[resource]
-		if !ok {
-			glog.Warningf("Resource %s not found", resource)
-			continue
-		}
-		// get group version resource
-		gv := schema.GroupVersion{Group: r.gvk.Group, Version: r.gvk.Version}
-		gvr := gv.WithResource(getResourceFromKind(r.gvk.Kind))
-		regResources = append(regResources, gvr)
-
-		decode := kscheme.Codecs.UniversalDeserializer().Decode
-		obj, _, err := decode([]byte(r.rawResource), nil, nil)
-		if err != nil {
-			glog.Warning("Unable to deocde")
-			continue
-		}
-		// create unstructured
-		rdata, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&obj)
-		if err != nil {
-			fmt.Println(err)
-			continue
-		}
-		unstr := unstructured.Unstructured{Object: rdata}
-		objects = append(objects, &unstr)
-	}
-	// new mock client
-	// Mock Client
-	c, err := dclient.NewMockClient(scheme, objects...)
-	if err != nil {
-		t.Error(err)
-	}
-	// set discovery Client
-	c.SetDiscovery(dclient.NewFakeDiscoveryClient(regResources))
-
-	return c
-}
-
-func (tb *testBundle) run(t *testing.T, testingapplyTest IApplyTest) {
-	glog.Infof("Start: test on test bundles %s", tb.path)
-	// run each scenario
-	for _, ts := range tb.scenarios {
-		// TODO create client only for generate
-		// If there are init resources defined then load them
-		c := tb.createClient(t, ts.InitResources)
-		// get policy
-		p, ok := tb.policies[ts.Policy]
-		if !ok {
-			glog.Warningf("Policy %s not found", ts.Policy)
-			continue
-		}
-		// get resources
-		r, ok := tb.resources[ts.Resource]
-		if !ok {
-			glog.Warningf("Resource %s not found", ts.Resource)
-			continue
-		}
-		// TODO: handle generate
-		if ts.Generation != nil {
-			// assuming its namespaces creation
-			decode := kscheme.Codecs.UniversalDeserializer().Decode
-			obj, _, err := decode([]byte(r.rawResource), nil, nil)
-			_, err = c.CreateResource(getResourceFromKind(r.gvk.Kind), "", obj)
-			if err != nil {
-				t.Errorf("error while creating namespace %s", ts.Resource)
-			}
-		}
-
-		mPatchedResource, mResult, vResult, err := testingapplyTest.applyPolicy(p, r, c)
-		if err != nil {
-			t.Error(err)
-		}
-		// check the expected scenario
-		tb.checkMutationResult(t, ts.Mutation, mPatchedResource, mResult)
-		tb.checkValidationResult(t, ts.Validation, vResult)
-		tb.checkGeneration(t, ts.Generation, c)
-	}
-	glog.Infof("Done: test on test bundles %s", tb.path)
-}
-
-func (tb *testBundle) checkGeneration(t *testing.T, expect *tGeneration, c *dclient.Client) {
-	if expect == nil {
-		glog.Info("No Generate check defined")
-		return
-	}
-	// iterate throught the expected resources and check if the client has them
-	for _, r := range expect.Resources {
-		_, err := c.GetResource(getResourceFromKind(r.Kind), r.Namespace, r.Name)
-		if err != nil {
-			t.Errorf("Resource %s/%s of kind %s not found", r.Namespace, r.Name, r.Kind)
-		}
-	}
-}
-
-func (tb *testBundle) checkValidationResult(t *testing.T, expect *tValidation, vResult result.Result) {
-	if expect == nil {
-		glog.Info("No Validation check defined")
-		return
-	}
-	// compare reason
-	if len(expect.Reason) > 0 && expect.Reason != vResult.GetReason().String() {
-		t.Error("Reason not matching")
-	}
-	// compare message
-	if len(expect.Message) > 0 && expect.Message != vResult.String() {
-		t.Error(("Message not matching"))
-	}
-}
-
-func (tb *testBundle) checkMutationResult(t *testing.T, expect *tMutation, pr *resourceInfo, mResult result.Result) {
-	if expect == nil {
-		glog.Info("No Mutation check defined")
-		return
-	}
-	// get expected patched resource
-	er, ok := tb.output[expect.MPatchedResource]
-	if !ok {
-		glog.Warningf("Resource %s not found", expect.MPatchedResource)
-		return
-	}
-	// compare patched resources
-	if !checkMutationRPatches(pr, er) {
-		fmt.Printf("Expected Resource %s \n", string(er.rawResource))
-		fmt.Printf("Patched Resource %s \n", string(pr.rawResource))
-
-		glog.Warningf("Expected resource %s ", string(pr.rawResource))
-		t.Error("Patched resources not as expected")
-	}
-	// compare reason
-	if len(expect.Reason) > 0 && expect.Reason != mResult.GetReason().String() {
-		t.Error("Reason not matching")
-	}
-	// compare message
-	if len(expect.Message) > 0 && expect.Message != mResult.String() {
-		t.Error(("Message not matching"))
-	}
-}

pkg/testutils/testsuite.go

@@ -1,65 +0,0 @@
-package testutils
-
-import (
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/golang/glog"
-)
-
-//NewTestSuite returns new test suite
-func NewTestSuite(t *testing.T, path string) *testSuite {
-	return &testSuite{
-		t:    t,
-		path: path,
-		tb:   []*testBundle{},
-	}
-}
-func (ts *testSuite) runTests() {
-	//TODO : make sure the implementation the interface is pointing to is not nil
-	if ts.applyTest == nil {
-		glog.Error("Apply Test set for the test suite")
-		return
-	}
-	// for each test bundle run the test scenario
-	for _, tb := range ts.tb {
-		tb.run(ts.t, ts.applyTest)
-	}
-}
-func (ts *testSuite) setApplyTest(applyTest IApplyTest) {
-	ts.applyTest = applyTest
-}
-
-type testSuite struct {
-	t         *testing.T
-	path      string
-	tb        []*testBundle
-	applyTest IApplyTest
-}
-
-func (ts *testSuite) buildTestSuite() error {
-	// loading test bundles for test suite
-	err := filepath.Walk(ts.path, func(path string, info os.FileInfo, err error) error {
-		if info.IsDir() {
-			glog.Infof("searching for test files at %s", path)
-			// check if there are resources dir and policies yaml
-			tb := NewTestBundle(path)
-			if tb != nil {
-				// try to load the test folder structure
-				err := tb.load()
-				if err != nil {
-					glog.Warningf("no supported test structure avaialbe at path %s", path)
-					return nil
-				}
-				glog.Infof("loading test suite at path %s", path)
-				ts.tb = append(ts.tb, tb)
-			}
-		}
-		return nil
-	})
-	if err != nil {
-		ts.t.Fatal(err)
-	}
-	return nil
-}

pkg/testutils/testutils_test.go

@@ -1,57 +0,0 @@
-package testutils
-
-import (
-	"testing"
-
-	"github.com/golang/glog"
-)
-
-// func TestExamples(t *testing.T) {
-// 	folders := []string{
-// 		"/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples",
-// 	}
-// 	testrunner(t, folders)
-// }
-
-func TestGenerate(t *testing.T) {
-	t.Skip("Under development")
-	folders := []string{
-		"/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/generate",
-	}
-	testrunner(t, folders)
-}
-
-func TestMutateOverlay(t *testing.T) {
-	t.Skip("Under development")
-	folders := []string{
-		"/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/mutate/overlay",
-	}
-	testrunner(t, folders)
-}
-
-func TestMutatePatches(t *testing.T) {
-	t.Skip("Under development")
-	folders := []string{
-		"/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/mutate/patches",
-	}
-	testrunner(t, folders)
-}
-
-func testrunner(t *testing.T, folders []string) {
-	for _, folder := range folders {
-		runTest(t, folder)
-	}
-}
-
-func runTest(t *testing.T, path string) {
-	// Load test suites at specified path
-	ts := LoadTestSuite(t, path)
-	// policy application logic
-	tp := &testPolicy{}
-	ts.setApplyTest(tp)
-	// run the tests for each test bundle
-	ts.runTests()
-	if ts != nil {
-		glog.Infof("Done running the test at %s", path)
-	}
-}

pkg/testutils/utils.go

@@ -1,310 +0,0 @@
-package testutils
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-	ospath "path"
-	"path/filepath"
-	"testing"
-
-	"gopkg.in/yaml.v2"
-
-	"github.com/golang/glog"
-	policytypes "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1"
-	client "github.com/nirmata/kyverno/pkg/dclient"
-	"github.com/nirmata/kyverno/pkg/engine"
-	"github.com/nirmata/kyverno/pkg/result"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	kyaml "k8s.io/apimachinery/pkg/util/yaml"
-	"k8s.io/client-go/kubernetes/scheme"
-)
-
-const (
-	defaultYamlSeparator = "---"
-)
-
-func loadFile(fileDir string) ([]byte, error) {
-	if _, err := os.Stat(fileDir); os.IsNotExist(err) {
-		return nil, err
-	}
-	return ioutil.ReadFile(fileDir)
-}
-
-func extractPolicy(fileDir string) (*policytypes.Policy, error) {
-	policy := &policytypes.Policy{}
-
-	file, err := loadFile(fileDir)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load file: %v", err)
-	}
-
-	policyBytes, err := kyaml.ToJSON(file)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := json.Unmarshal(policyBytes, policy); err != nil {
-		return nil, fmt.Errorf("failed to decode policy %s, err: %v", policy.Name, err)
-	}
-
-	if policy.TypeMeta.Kind != "Policy" {
-		return nil, fmt.Errorf("failed to parse policy")
-	}
-
-	return policy, nil
-}
-
-type resourceInfo struct {
-	rawResource []byte
-	gvk         *metav1.GroupVersionKind
-}
-
-func (ri resourceInfo) isSame(other resourceInfo) bool {
-	// compare gvk
-	if *ri.gvk != *other.gvk {
-		return false
-	}
-	// compare rawResource
-	return bytes.Equal(ri.rawResource, other.rawResource)
-}
-
-func getResourceYAML(d []byte) {
-	// fmt.Println(string(d))
-	// convert json to yaml
-	// print the result for reference
-	// can be used as a dry run the get the expected result
-}
-
-func extractResourceRaw(d []byte) (string, *resourceInfo) {
-	decode := scheme.Codecs.UniversalDeserializer().Decode
-	obj, gvk, err := decode([]byte(d), nil, nil)
-	if err != nil {
-		glog.Warningf("Error while decoding YAML object, err: %s\n", err)
-		return "", nil
-	}
-	raw, err := json.Marshal(obj)
-	if err != nil {
-		glog.Warningf("Error while marshalling manifest, err: %v\n", err)
-		return "", nil
-	}
-	gvkInfo := &metav1.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind}
-	rn := ParseNameFromObject(raw)
-	rns := ParseNamespaceFromObject(raw)
-	if rns != "" {
-		rn = rns + "/" + rn
-	}
-	return rn, &resourceInfo{rawResource: raw, gvk: gvkInfo}
-}
-
-func extractResource(resource string) (map[string]*resourceInfo, error) {
-	resources := make(map[string]*resourceInfo)
-	data, err := loadFile(resource)
-	if err != nil {
-		glog.Warningf("Error while loading file: %v\n", err)
-		return nil, err
-	}
-	dd := bytes.Split(data, []byte(defaultYamlSeparator))
-	for _, d := range dd {
-		rn, r := extractResourceRaw(d)
-		resources[rn] = r
-	}
-	return resources, nil
-}
-
-func ParseApiVersionFromObject(bytes []byte) string {
-	var objectJSON map[string]interface{}
-	json.Unmarshal(bytes, &objectJSON)
-	if apiVersion, ok := objectJSON["apiVersion"].(string); ok {
-		return apiVersion
-	}
-	return ""
-}
-
-func ParseKindFromObject(bytes []byte) string {
-	var objectJSON map[string]interface{}
-	json.Unmarshal(bytes, &objectJSON)
-	if kind, ok := objectJSON["kind"].(string); ok {
-		return kind
-	}
-	return ""
-}
-
-//ParseNameFromObject extracts resource name from JSON obj
-func ParseNameFromObject(bytes []byte) string {
-	var objectJSON map[string]interface{}
-	json.Unmarshal(bytes, &objectJSON)
-
-	meta := objectJSON["metadata"].(map[string]interface{})
-
-	if name, ok := meta["name"].(string); ok {
-		return name
-	}
-	return ""
-}
-
-// ParseNamespaceFromObject extracts the namespace from the JSON obj
-func ParseNamespaceFromObject(bytes []byte) string {
-	var objectJSON map[string]interface{}
-	json.Unmarshal(bytes, &objectJSON)
-
-	meta := objectJSON["metadata"].(map[string]interface{})
-
-	if namespace, ok := meta["namespace"].(string); ok {
-		return namespace
-	}
-	return ""
-}
-
-type IApplyTest interface {
-	applyPolicy(policy *policytypes.Policy, resource *resourceInfo, client *client.Client) (*resourceInfo, result.Result, result.Result, error)
-}
-
-type testPolicy struct {
-}
-
-func (tp *testPolicy) applyPolicy(policy *policytypes.Policy, resource *resourceInfo, client *client.Client) (*resourceInfo, result.Result, result.Result, error) {
-	// apply policy on the trigger resource
-	// Mutate
-	var vResult result.Result
-	var patchedResource []byte
-	mPatches, mResult := engine.Mutate(*policy, resource.rawResource, *resource.gvk)
-	// TODO: only validate if there are no errors in mutate, why?
-	err := mResult.ToError()
-	if err == nil && len(mPatches) != 0 {
-		patchedResource, err = engine.ApplyPatches(resource.rawResource, mPatches)
-		if err != nil {
-			return nil, nil, nil, err
-		}
-		// Validate
-		vResult = engine.Validate(*policy, patchedResource, *resource.gvk)
-	}
-	// Generate
-	if client != nil {
-		engine.Generate(client, *policy, resource.rawResource, *resource.gvk)
-	}
-
-	// transform the patched Resource into resource Info
-	_, ri := extractResourceRaw(patchedResource)
-	// return the results
-	return ri, mResult, vResult, nil
-	// TODO: merge the results for mutation and validation
-}
-
-type tScenario struct {
-	Policy        string       `yaml:"policy"`
-	Resource      string       `yaml:"resource"`
-	InitResources []string     `yaml:"initResources,omitempty"`
-	Mutation      *tMutation   `yaml:"mutation,omitempty"`
-	Validation    *tValidation `yaml:"validation,omitempty"`
-	Generation    *tGeneration `yaml:"generation,omitempty"`
-}
-
-type tGeneration struct {
-	Resources []tResource `yaml:"resource"`
-}
-
-type tResource struct {
-	Name      string `yaml:"name"`
-	Namespace string `yaml:"namespace,omitempty"`
-	Kind      string `yaml:"kind"`
-}
-
-type tValidation struct {
-	Reason  string `yaml:"reason,omitempty"`
-	Message string `yaml:"message,omitempty"`
-	Error   string `yaml:"error,omitempty"`
-}
-
-type tMutation struct {
-	MPatchedResource string `yaml:"mPatchedResource,omitempty"`
-	Reason           string `yaml:"reason,omitempty"`
-	Message          string `yaml:"message,omitempty"`
-	Error            string `yaml:"error,omitempty"`
-}
-
-func LoadScenarios(file string) ([]*tScenario, error) {
-	ts := []*tScenario{}
-	// read the file
-	data, err := loadFile(file)
-	if err != nil {
-		glog.Warningf("Error while loading file: %v\n", err)
-		return nil, err
-	}
-	dd := bytes.Split(data, []byte(defaultYamlSeparator))
-	for _, d := range dd {
-		s := &tScenario{}
-		err := yaml.Unmarshal([]byte(d), s)
-		if err != nil {
-			glog.Warningf("Error while decoding YAML object, err: %s", err)
-			continue
-		}
-		ts = append(ts, s)
-	}
-	return ts, nil
-}
-
-// Load policy & resource files
-// engine pass the (policy, resource)
-// check the expected response
-
-const examplesPath string = "examples"
-const resourcesFolder string = "resources"
-const tScenarioFile string = "testScenarios.yaml"
-const outputFolder string = "output"
-
-//LoadTestSuite  reads the resource, policy and scenario files
-func LoadTestSuite(t *testing.T, path string) *testSuite {
-	glog.Infof("loading test suites at %s", path)
-	// gp := os.Getenv("GOPATH")
-	// ap := ospath.Join(gp, "src/github.com/nirmata/kyverno")
-	// build test suite
-	// each suite contains test bundles for test sceanrios
-	// ts := NewTestSuite(t, ospath.Join(ap, examplesPath))
-	ts := NewTestSuite(t, path)
-	ts.buildTestSuite()
-	glog.Infof("done loading test suite at %s", path)
-	return ts
-}
-
-func checkMutationRPatches(er *resourceInfo, pr *resourceInfo) bool {
-	if !er.isSame(*pr) {
-		getResourceYAML(pr.rawResource)
-		return false
-	}
-	return true
-}
-
-func getYAMLfiles(path string) (yamls []string) {
-	fileInfo, err := ioutil.ReadDir(path)
-	if err != nil {
-		return nil
-	}
-	for _, file := range fileInfo {
-		if file.Name() == tScenarioFile {
-			continue
-		}
-		if filepath.Ext(file.Name()) == ".yml" || filepath.Ext(file.Name()) == ".yaml" {
-			yamls = append(yamls, ospath.Join(path, file.Name()))
-		}
-	}
-	return yamls
-}
-
-var kindToResource = map[string]string{
-	"ConfigMap":     "configmaps",
-	"Endpoints":     "endpoints",
-	"Namespace":     "namespaces",
-	"Secret":        "secrets",
-	"Deployment":    "deployments",
-	"NetworkPolicy": "networkpolicies",
-}
-
-func getResourceFromKind(kind string) string {
-	if resource, ok := kindToResource[kind]; ok {
-		return resource
-	}
-	return ""
-}

test/output/cm_copied_cm.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+data:
+  configmap.data: |
+    ns=default
+    labels=originalLabel
+    labelscount=1
+  game.properties: |
+    enemies=predators
+    lives=3
+  ui.properties: "color.good=green\ncolor.bad=red \n"
+kind: ConfigMap
+metadata:
+  labels:
+    originalLabel: isHere
+  name: copied-cm
+  namespace: ns2

test/output/cm_default_config.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+data:
+  configmap.data: |
+    ns=default
+    labels=originalLabel
+    labelscount=1
+  game.properties: |
+    enemies=predators
+    lives=3
+  ui.properties: "color.good=green\ncolor.bad=red \n"
+kind: ConfigMap
+metadata:
+  labels:
+    originalLabel: isHere
+  name: default-config
+  namespace: ns2

test/output/cm_zk-kafka-address.yaml

@@ -0,0 +1,8 @@
+data:
+  KAFKA_ADDRESS: 192.168.10.13:9092,192.168.10.14:9092,192.168.10.15:9092
+  ZK_ADDRESS: 192.168.10.10:2181,192.168.10.11:2181,192.168.10.12:2181
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: zk-kafka-address
+  namespace: ns2

test/output/np_deny-all-traffic.yaml

@@ -0,0 +1,13 @@
+metadata:
+  annotations: {}
+  labels:
+    policyname: default
+  name: deny-all-traffic
+  namespace: ns2
+podSelector:
+  matchExpressions: []
+  matchLabels: {}
+policyTypes: []
+spec: 
+kind: NetworkPolicy
+apiVersion: extensions/v1beta1

test/output/sc_mongo_cred.yaml

@@ -0,0 +1,8 @@
+data:
+  DB_PASSWORD: YXBwc3dvcmQ=
+  DB_USER: YWJyYWthZGFicmE=
+metadata:
+  name: mongo-creds
+  namespace: ns2
+kind: Secret
+apiVersion: v1

test/scenarios/cli.yaml

@@ -0,0 +1,20 @@
+# file path relative to project root
+input:
+  policy: examples/cli/policy_deployment.yaml
+  resource: examples/cli/nginx.yaml
+expected:
+  mutation:
+    patched_resource: test/output/nginx.yaml
+    reason: Success
+  validation:
+    reason: Success
+---
+input:
+  policy: examples/cli/policy_deployment.yaml
+  resource: examples/cli/ghost.yaml
+expected:
+  mutation:
+    patched_resource: test/output/ghost.yaml
+    reason: Success
+  validation:
+    reason: Success

test/scenarios/generate.yaml

@@ -0,0 +1,30 @@
+# file path relative to project root
+input:
+  policy: examples/generate/policy_basic.yaml
+  resource: examples/generate/namespace.yaml
+  load_resources:
+    - examples/generate/configMap_default.yaml
+expected:
+  generation:
+    resources:
+      - test/output/cm_default_config.yaml
+      - test/output/sc_mongo_cred.yaml
+---
+input:
+  policy: examples/generate/policy_generate.yaml
+  resource: examples/generate/namespace.yaml
+  load_resources:
+    - examples/generate/configMap.yaml
+expected:
+  generation:
+    resources:
+      - test/output/cm_copied_cm.yaml
+      - test/output/cm_zk-kafka-address.yaml
+---
+input:
+  policy: examples/generate/policy_networkPolicy.yaml
+  resource: examples/generate/namespace.yaml
+expected:
+  generation:
+    resources:
+      - test/output/np_deny-all-traffic.yaml

test/scenarios/mutate/overlay.yaml

@@ -0,0 +1,8 @@
+# file path relative to project root
+input:
+  policy: examples/mutate/overlay/policy_imagePullPolicy.yaml
+  resource: examples/mutate/overlay/nginx.yaml
+expected:
+  mutation:
+    patched_resource: test/output/op_overlay_nginx.yaml
+    reason: Success

test/scenarios/mutate/patches.yaml

@@ -0,0 +1,8 @@
+# file path relative to project root
+input:
+  policy: examples/mutate/patches/policy_endpoints.yaml
+  resource: examples/mutate/patches/endpoints.yaml
+expected:
+  mutation:
+    patched_resource: test/output/op_patches_endpoints.yaml
+    reason: Success