From 4692058a4a399306bef11e66398e3ab194d9d7f4 Mon Sep 17 00:00:00 2001 From: shivdudhani Date: Wed, 12 Jun 2019 16:18:31 -0700 Subject: [PATCH 01/10] initial commit --- .../mutate/patches/resources/endpoints.yaml | 13 ++ examples/mutate/patches/testScenarios.yaml | 15 ++ pkg/testutils/testbundle.go | 156 +++++++++++++ pkg/testutils/testsuite.go | 66 ++++++ pkg/testutils/testutils.go | 54 +++++ pkg/testutils/testutils_test.go | 20 ++ pkg/testutils/utils.go | 216 ++++++++++++++++++ 7 files changed, 540 insertions(+) create mode 100644 examples/mutate/patches/resources/endpoints.yaml create mode 100644 examples/mutate/patches/testScenarios.yaml create mode 100644 pkg/testutils/testbundle.go create mode 100644 pkg/testutils/testsuite.go create mode 100644 pkg/testutils/testutils.go create mode 100644 pkg/testutils/testutils_test.go create mode 100644 pkg/testutils/utils.go diff --git a/examples/mutate/patches/resources/endpoints.yaml b/examples/mutate/patches/resources/endpoints.yaml new file mode 100644 index 0000000000..958d931482 --- /dev/null +++ b/examples/mutate/patches/resources/endpoints.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Endpoints +metadata: + name: test-endpoint + labels: + label : test +subsets: +- addresses: + - ip: 192.168.10.171 + ports: + - name: secure-connection + port: 443 + protocol: TCP diff --git a/examples/mutate/patches/testScenarios.yaml b/examples/mutate/patches/testScenarios.yaml new file mode 100644 index 0000000000..7f0fec053a --- /dev/null +++ b/examples/mutate/patches/testScenarios.yaml @@ -0,0 +1,15 @@ +# input +policy: policy-endpoints +resource: test-endpoint +initResources: +# expected +mutation: + mPatchedResource: test-endpoint + reason: Success +validation: + reason: Success +--- +policy: policy-endpoints +resource: test-endpoint +initResources: +mPatchedResource: test-endpoint \ No newline at end of file diff --git a/pkg/testutils/testbundle.go b/pkg/testutils/testbundle.go new file mode 100644 index 0000000000..4cec7c06c2 --- /dev/null +++ b/pkg/testutils/testbundle.go @@ -0,0 +1,156 @@ +package testutils + +import ( + "fmt" + "os" + ospath "path" + "testing" + + "github.com/golang/glog" + policytypes "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" + "github.com/nirmata/kyverno/pkg/result" +) + +func NewTestBundle(path string) *testBundle { + return &testBundle{ + path: path, + policies: make(map[string]*policytypes.Policy), + resources: make(map[string]*resourceInfo), + initResources: make(map[string]*resourceInfo), + } +} + +func (tb *testBundle) load() error { + // check if resources folder is defined + rpath := ospath.Join(tb.path, resourcesFolder) + _, err := os.Stat(rpath) + if os.IsNotExist(err) { + glog.Warningf("Resources directory not present at %s", tb.path) + return fmt.Errorf("Resources directory not present at %s", tb.path) + } + + // check if scenario yaml is defined + spath := ospath.Join(tb.path, tScenarioFile) + _, err = os.Stat(spath) + if os.IsNotExist(err) { + return fmt.Errorf("Scenario file %s not defined at %s", tScenarioFile, tb.path) + } + tb.scenarios, err = LoadScenarios(spath) + if err != nil { + return err + } + // check if there are any files + pYAMLs := getYAMLfiles(tb.path) + if len(pYAMLs) == 0 { + return fmt.Errorf("No policy yaml found at path %s", tb.path) + } + for _, p := range pYAMLs { + // extract policy + policy, err := extractPolicy(p) + if err != nil { + glog.Errorf("unable to extract policy: %s", err) + continue + } + tb.policies[policy.GetName()] = policy + } + + // extract resources + rYAMLs := getYAMLfiles(rpath) + if len(rYAMLs) == 0 { + return fmt.Errorf("No resource yaml found at path %s", rpath) + } + for _, r := range rYAMLs { + resources, err := extractResource(r) + if err != nil { + glog.Errorf("unable to extract resource: %s", err) + } + tb.mergeResources(resources) + } + return nil +} + +func (tb *testBundle) mergeResources(rs map[string]*resourceInfo) { + for k, v := range rs { + if _, ok := tb.resources[k]; ok { + glog.Infof("resource already defined %s ", k) + continue + } + tb.resources[k] = v + } +} + +type testBundle struct { + path string + policies map[string]*policytypes.Policy + resources map[string]*resourceInfo + initResources map[string]*resourceInfo + scenarios []*tScenario +} + +func (tb *testBundle) run(t *testing.T, testingapplyTest IApplyTest) { + // run each scenario + for _, ts := range tb.scenarios { + // get policy + p, ok := tb.policies[ts.Policy] + if !ok { + glog.Warningf("Policy %s not found", ts.Policy) + continue + } + // get resources + r, ok := tb.resources[ts.Resource] + if !ok { + glog.Warningf("Resource %s not found", ts.Resource) + continue + } + // TODO: handle generate + mPatchedResource, mResult, vResult, err := testingapplyTest.applyPolicy(p, r, nil) + if err != nil { + t.Fatal(err) + } + // check the expected scenario + tb.checkMutationResult(t, ts.Mutation, mPatchedResource, mResult) + tb.checkValidationResult(t, ts.Validation, vResult) + } +} + +func (tb *testBundle) checkValidationResult(t *testing.T, expect *tValidation, vResult result.Result) { + if expect == nil { + glog.Info("No Validation check defined") + return + } + // compare result + // compare reason + if len(expect.Reason) > 0 && expect.Reason != vResult.GetReason().String() { + t.Error("Reason not matching") + } + // compare message + if len(expect.Message) > 0 && expect.Message != vResult.String() { + t.Error(("Message not matching")) + } +} + +func (tb *testBundle) checkMutationResult(t *testing.T, expect *tMutation, pr *resourceInfo, mResult result.Result) { + if expect == nil { + glog.Info("No Mutation check defined") + return + } + // get expected patched resource + pr, ok := tb.resources[expect.MPatchedResource] + if !ok { + glog.Warningf("Resource %s not found", expect.MPatchedResource) + return + } + // compare patched resources + if !checkMutationRPatches(pr, pr) { + t.Error("Patched resources not as expected") + } + // compare result + // compare reason + if len(expect.Reason) > 0 && expect.Reason != mResult.GetReason().String() { + t.Error("Reason not matching") + } + // compare message + if len(expect.Message) > 0 && expect.Message != mResult.String() { + t.Error(("Message not matching")) + } +} diff --git a/pkg/testutils/testsuite.go b/pkg/testutils/testsuite.go new file mode 100644 index 0000000000..7240ace08e --- /dev/null +++ b/pkg/testutils/testsuite.go @@ -0,0 +1,66 @@ +package testutils + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/golang/glog" +) + +//NewTestSuite returns new test suite +func NewTestSuite(t *testing.T, path string) *testSuite { + return &testSuite{ + t: t, + path: path, + tb: []*testBundle{}, + } +} +func (ts *testSuite) runTests() { + //TODO : make sure the implementation the interface is pointing to is not nil + if ts.applyTest == nil { + glog.Error("Apply Test set for the test suite") + return + } + // for each test bundle run the test scenario + for _, tb := range ts.tb { + tb.run(ts.t, ts.applyTest) + } +} +func (ts *testSuite) setApplyTest(applyTest IApplyTest) { + ts.applyTest = applyTest +} + +type testSuite struct { + t *testing.T + path string + tb []*testBundle + applyTest IApplyTest +} + +func (ts *testSuite) buildTestSuite() error { + // loading test bundles for test suite + fmt.Println(ts.path) + err := filepath.Walk(ts.path, func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + // check if there are resources dir and policies yaml + tb := NewTestBundle(path) + if tb != nil { + // load resources + err := tb.load() + if err != nil { + // glog.Error(err) + return nil + } + ts.tb = append(ts.tb, tb) + // fmt.Println(path) + } + } + return nil + }) + if err != nil { + ts.t.Fatal(err) + } + return nil +} diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go new file mode 100644 index 0000000000..8f2353169e --- /dev/null +++ b/pkg/testutils/testutils.go @@ -0,0 +1,54 @@ +package testutils + +import ( + "fmt" + "io/ioutil" + ospath "path" + "path/filepath" + "testing" +) + +// Load policy & resource files +// engine pass the (policy, resource) +// check the expected response + +const examplesPath string = "examples" +const resourcesFolder string = "resources" +const tScenarioFile string = "testScenarios.yaml" + +//LoadTestSuite reads the resource, policy and scenario files +func LoadTestSuite(t *testing.T, path string) *testSuite { + // gp := os.Getenv("GOPATH") + // ap := ospath.Join(gp, "src/github.com/nirmata/kyverno") + // build test suite + // each suite contains test bundles for test sceanrios + // ts := NewTestSuite(t, ospath.Join(ap, examplesPath)) + ts := NewTestSuite(t, path) + ts.buildTestSuite() + return ts +} + +func checkMutationRPatches(er *resourceInfo, pr *resourceInfo) bool { + if !er.isSame(*pr) { + getResourceYAML(pr.rawResource) + return false + } + return true +} + +func getYAMLfiles(path string) (yamls []string) { + fileInfo, err := ioutil.ReadDir(path) + if err != nil { + return nil + } + for _, file := range fileInfo { + fmt.Println(filepath.Ext(file.Name())) + if file.Name() == tScenarioFile { + continue + } + if filepath.Ext(file.Name()) == ".yml" || filepath.Ext(file.Name()) == ".yaml" { + yamls = append(yamls, ospath.Join(path, file.Name())) + } + } + return yamls +} diff --git a/pkg/testutils/testutils_test.go b/pkg/testutils/testutils_test.go new file mode 100644 index 0000000000..cd0645de4a --- /dev/null +++ b/pkg/testutils/testutils_test.go @@ -0,0 +1,20 @@ +package testutils + +import ( + "fmt" + "testing" +) + +func TestUtils(t *testing.T) { + file := "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/mutate/patches" + ts := LoadTestSuite(t, file) + // policy application logic + tp := &testPolicy{} + ts.setApplyTest(tp) + // run the tests for each test bundle + ts.runTests() + if ts != nil { + fmt.Println("Done building the test bundles") + } + // run the tests against the policy engine +} diff --git a/pkg/testutils/utils.go b/pkg/testutils/utils.go new file mode 100644 index 0000000000..a2c41aa416 --- /dev/null +++ b/pkg/testutils/utils.go @@ -0,0 +1,216 @@ +package testutils + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "os" + + "gopkg.in/yaml.v2" + + "github.com/golang/glog" + policytypes "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" + client "github.com/nirmata/kyverno/pkg/dclient" + "github.com/nirmata/kyverno/pkg/engine" + "github.com/nirmata/kyverno/pkg/result" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes/scheme" +) + +const ( + defaultYamlSeparator = "---" +) + +func loadFile(fileDir string) ([]byte, error) { + if _, err := os.Stat(fileDir); os.IsNotExist(err) { + return nil, err + } + return ioutil.ReadFile(fileDir) +} + +func extractPolicy(fileDir string) (*policytypes.Policy, error) { + policy := &policytypes.Policy{} + + file, err := loadFile(fileDir) + if err != nil { + return nil, fmt.Errorf("failed to load file: %v", err) + } + + policyBytes, err := kyaml.ToJSON(file) + if err != nil { + return nil, err + } + + if err := json.Unmarshal(policyBytes, policy); err != nil { + return nil, fmt.Errorf("failed to decode policy %s, err: %v", policy.Name, err) + } + + if policy.TypeMeta.Kind != "Policy" { + return nil, fmt.Errorf("failed to parse policy") + } + + return policy, nil +} + +type resourceInfo struct { + rawResource []byte + gvk *metav1.GroupVersionKind +} + +func (ri resourceInfo) isSame(other resourceInfo) bool { + // compare gvk + if ri.gvk != other.gvk { + return false + } + // compare rawResource + return bytes.Equal(ri.rawResource, other.rawResource) +} + +func getResourceYAML(d []byte) { + fmt.Println(string(d)) + // convert json to yaml + // print the result for reference + // can be used as a dry run the get the expected result +} + +func extractResourceRaw(d []byte) (string, *resourceInfo) { + decode := scheme.Codecs.UniversalDeserializer().Decode + obj, gvk, err := decode([]byte(d), nil, nil) + if err != nil { + glog.Warningf("Error while decoding YAML object, err: %s\n", err) + return "", nil + } + raw, err := json.Marshal(obj) + if err != nil { + glog.Warningf("Error while marshalling manifest, err: %v\n", err) + return "", nil + } + gvkInfo := &metav1.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind} + rn := ParseNameFromObject(raw) + rns := ParseNamespaceFromObject(raw) + if rns != "" { + rn = rns + "/" + rn + } + return rn, &resourceInfo{rawResource: raw, gvk: gvkInfo} +} + +func extractResource(resource string) (map[string]*resourceInfo, error) { + resources := make(map[string]*resourceInfo) + data, err := loadFile(resource) + if err != nil { + glog.Warningf("Error while loading file: %v\n", err) + return nil, err + } + dd := bytes.Split(data, []byte(defaultYamlSeparator)) + for _, d := range dd { + rn, r := extractResourceRaw(d) + resources[rn] = r + } + return resources, nil +} + +//ParseNameFromObject extracts resource name from JSON obj +func ParseNameFromObject(bytes []byte) string { + var objectJSON map[string]interface{} + json.Unmarshal(bytes, &objectJSON) + + meta := objectJSON["metadata"].(map[string]interface{}) + + if name, ok := meta["name"].(string); ok { + return name + } + return "" +} + +// ParseNamespaceFromObject extracts the namespace from the JSON obj +func ParseNamespaceFromObject(bytes []byte) string { + var objectJSON map[string]interface{} + json.Unmarshal(bytes, &objectJSON) + + meta := objectJSON["metadata"].(map[string]interface{}) + + if namespace, ok := meta["namespace"].(string); ok { + return namespace + } + return "" +} + +type IApplyTest interface { + applyPolicy(policy *policytypes.Policy, resource *resourceInfo, client *client.Client) (*resourceInfo, result.Result, result.Result, error) +} + +type testPolicy struct { +} + +func (tp *testPolicy) applyPolicy(policy *policytypes.Policy, resource *resourceInfo, client *client.Client) (*resourceInfo, result.Result, result.Result, error) { + // apply policy on the trigger resource + // Mutate + var vResult result.Result + var patchedResource []byte + mPatches, mResult := engine.Mutate(*policy, resource.rawResource, *resource.gvk) + // TODO: only validate if there are no errors in mutate, why? + err := mResult.ToError() + if err == nil && len(mPatches) != 0 { + patchedResource, err = engine.ApplyPatches(resource.rawResource, mPatches) + if err != nil { + return nil, nil, nil, err + } + // Validate + vResult = engine.Validate(*policy, patchedResource, *resource.gvk) + } + // Generate + if client == nil { + glog.Warning("Client is required to test generate") + } + // transform the patched Resource into resource Info + _, ri := extractResourceRaw(patchedResource) + // return the results + return ri, mResult, vResult, nil + // TODO: merge the results for mutation and validation +} + +type tScenario struct { + Policy string `yaml:"policy"` + Resource string `yaml:"resource"` + InitResources []string `yaml:"initResources,omitempty"` + Mutation *tMutation `yaml:"mutation,omitempty"` + Validation *tValidation `yaml:"validation,omitempty"` +} + +type tValidation struct { + Reason string `yaml:"reason,omitempty"` + Message string `yaml:"message,omitempty"` + Error string `yaml:"error,omitempty"` +} + +type tMutation struct { + MPatchedResource string `yaml:"mPatchedResource,omitempty"` + Reason string `yaml:"reason,omitempty"` + Message string `yaml:"message,omitempty"` + Error string `yaml:"error,omitempty"` +} + +func LoadScenarios(file string) ([]*tScenario, error) { + ts := []*tScenario{} + // read the file + data, err := loadFile(file) + if err != nil { + glog.Warningf("Error while loading file: %v\n", err) + return nil, err + } + dd := bytes.Split(data, []byte(defaultYamlSeparator)) + for _, d := range dd { + s := &tScenario{} + err := yaml.Unmarshal([]byte(d), s) + if err != nil { + glog.Warningf("Error while decoding YAML object, err: %s", err) + continue + } + fmt.Println(s.Policy) + fmt.Println(s.Resource) + ts = append(ts, s) + } + return ts, nil +} From 711a892c12563de22b6446795e683e3ff244b8a9 Mon Sep 17 00:00:00 2001 From: shivdudhani Date: Thu, 13 Jun 2019 00:21:00 -0700 Subject: [PATCH 02/10] add output support --- .../patches/{ => output}/endpoints.yaml | 0 pkg/testutils/testbundle.go | 134 +++++++++++++----- pkg/testutils/testutils.go | 1 + pkg/testutils/utils.go | 5 + 4 files changed, 101 insertions(+), 39 deletions(-) rename examples/mutate/patches/{ => output}/endpoints.yaml (100%) diff --git a/examples/mutate/patches/endpoints.yaml b/examples/mutate/patches/output/endpoints.yaml similarity index 100% rename from examples/mutate/patches/endpoints.yaml rename to examples/mutate/patches/output/endpoints.yaml diff --git a/pkg/testutils/testbundle.go b/pkg/testutils/testbundle.go index 4cec7c06c2..b407fa97a0 100644 --- a/pkg/testutils/testbundle.go +++ b/pkg/testutils/testbundle.go @@ -1,6 +1,7 @@ package testutils import ( + "bytes" "fmt" "os" ospath "path" @@ -9,36 +10,98 @@ import ( "github.com/golang/glog" policytypes "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" "github.com/nirmata/kyverno/pkg/result" + "gopkg.in/yaml.v2" ) func NewTestBundle(path string) *testBundle { return &testBundle{ - path: path, - policies: make(map[string]*policytypes.Policy), - resources: make(map[string]*resourceInfo), - initResources: make(map[string]*resourceInfo), + path: path, + policies: make(map[string]*policytypes.Policy), + resources: make(map[string]*resourceInfo), + output: make(map[string]*resourceInfo), } } -func (tb *testBundle) load() error { - // check if resources folder is defined - rpath := ospath.Join(tb.path, resourcesFolder) - _, err := os.Stat(rpath) +func loadResources(tbPath string, rs map[string]*resourceInfo, file string) { + path := ospath.Join(tbPath, file) + _, err := os.Stat(path) if os.IsNotExist(err) { - glog.Warningf("Resources directory not present at %s", tb.path) - return fmt.Errorf("Resources directory not present at %s", tb.path) + glog.Warningf("%s directory not present at %s", file, tbPath) + return + } + // Load the resources from the output folder + yamls := getYAMLfiles(path) + if len(yamls) == 0 { + glog.Warningf("No resource yaml found at path %s", path) + return + } + for _, r := range yamls { + resources, err := extractResource(r) + if err != nil { + glog.Errorf("unable to extract resource: %s", err) + } + mergeResources(rs, resources) + } +} + +func (tb *testBundle) loadOutput() { + // check if output folder is defined + opath := ospath.Join(tb.path, outputFolder) + _, err := os.Stat(opath) + if os.IsNotExist(err) { + glog.Warningf("Output directory not present at %s", tb.path) + return + } + // Load the resources from the output folder + oYAMLs := getYAMLfiles(opath) + if len(oYAMLs) == 0 { + glog.Warningf("No resource yaml found at path %s", opath) + return } - // check if scenario yaml is defined - spath := ospath.Join(tb.path, tScenarioFile) - _, err = os.Stat(spath) - if os.IsNotExist(err) { - return fmt.Errorf("Scenario file %s not defined at %s", tScenarioFile, tb.path) + for _, r := range oYAMLs { + resources, err := extractResource(r) + if err != nil { + glog.Errorf("unable to extract resource: %s", err) + } + mergeResources(tb.output, resources) } - tb.scenarios, err = LoadScenarios(spath) +} + +func loadScenarios(tbPath string, file string) ([]*tScenario, error) { + // check if scenario yaml is defined + spath := ospath.Join(tbPath, file) + _, err := os.Stat(spath) + if os.IsNotExist(err) { + return nil, fmt.Errorf("Scenario file %s not defined at %s", file, tbPath) + } + ts := []*tScenario{} + // read the file + data, err := loadFile(spath) + if err != nil { + glog.Warningf("Error while loading file: %v\n", err) + return nil, err + } + dd := bytes.Split(data, []byte(defaultYamlSeparator)) + for _, d := range dd { + s := &tScenario{} + err := yaml.Unmarshal([]byte(d), s) + if err != nil { + glog.Warningf("Error while decoding YAML object, err: %s", err) + continue + } + fmt.Println(s.Policy) + fmt.Println(s.Resource) + ts = append(ts, s) + } + return ts, nil +} +func (tb *testBundle) load() error { + scenarios, err := loadScenarios(tb.path, tScenarioFile) if err != nil { return err } + tb.scenarios = scenarios // check if there are any files pYAMLs := getYAMLfiles(tb.path) if len(pYAMLs) == 0 { @@ -54,37 +117,30 @@ func (tb *testBundle) load() error { tb.policies[policy.GetName()] = policy } - // extract resources - rYAMLs := getYAMLfiles(rpath) - if len(rYAMLs) == 0 { - return fmt.Errorf("No resource yaml found at path %s", rpath) - } - for _, r := range rYAMLs { - resources, err := extractResource(r) - if err != nil { - glog.Errorf("unable to extract resource: %s", err) - } - tb.mergeResources(resources) - } + // load resources + loadResources(tb.path, tb.resources, resourcesFolder) + // load output resources + loadResources(tb.path, tb.output, outputFolder) + return nil } -func (tb *testBundle) mergeResources(rs map[string]*resourceInfo) { - for k, v := range rs { - if _, ok := tb.resources[k]; ok { +func mergeResources(rs map[string]*resourceInfo, other map[string]*resourceInfo) { + for k, v := range other { + if _, ok := rs[k]; ok { glog.Infof("resource already defined %s ", k) continue } - tb.resources[k] = v + rs[k] = v } } type testBundle struct { - path string - policies map[string]*policytypes.Policy - resources map[string]*resourceInfo - initResources map[string]*resourceInfo - scenarios []*tScenario + path string + policies map[string]*policytypes.Policy + resources map[string]*resourceInfo + output map[string]*resourceInfo + scenarios []*tScenario } func (tb *testBundle) run(t *testing.T, testingapplyTest IApplyTest) { @@ -135,13 +191,13 @@ func (tb *testBundle) checkMutationResult(t *testing.T, expect *tMutation, pr *r return } // get expected patched resource - pr, ok := tb.resources[expect.MPatchedResource] + er, ok := tb.resources[expect.MPatchedResource] if !ok { glog.Warningf("Resource %s not found", expect.MPatchedResource) return } // compare patched resources - if !checkMutationRPatches(pr, pr) { + if !checkMutationRPatches(pr, er) { t.Error("Patched resources not as expected") } // compare result diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 8f2353169e..1d6d104d17 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -15,6 +15,7 @@ import ( const examplesPath string = "examples" const resourcesFolder string = "resources" const tScenarioFile string = "testScenarios.yaml" +const outputFolder string = "output" //LoadTestSuite reads the resource, policy and scenario files func LoadTestSuite(t *testing.T, path string) *testSuite { diff --git a/pkg/testutils/utils.go b/pkg/testutils/utils.go index a2c41aa416..ccf4de6e15 100644 --- a/pkg/testutils/utils.go +++ b/pkg/testutils/utils.go @@ -154,16 +154,21 @@ func (tp *testPolicy) applyPolicy(policy *policytypes.Policy, resource *resource err := mResult.ToError() if err == nil && len(mPatches) != 0 { patchedResource, err = engine.ApplyPatches(resource.rawResource, mPatches) + fmt.Println(len(resource.rawResource)) + fmt.Println(len(patchedResource)) if err != nil { return nil, nil, nil, err } // Validate vResult = engine.Validate(*policy, patchedResource, *resource.gvk) + fmt.Println(len(patchedResource)) } // Generate if client == nil { glog.Warning("Client is required to test generate") } + fmt.Println(len(patchedResource)) + // transform the patched Resource into resource Info _, ri := extractResourceRaw(patchedResource) // return the results From 6b91ce18d2aaa8b9442e0f121468a40c23c89471 Mon Sep 17 00:00:00 2001 From: shivdudhani Date: Thu, 13 Jun 2019 12:01:03 -0700 Subject: [PATCH 03/10] scan examples folder to build TS --- examples/cli/output/ghost.yaml | 40 +++++++++++++++++++ examples/cli/output/nginx.yaml | 29 ++++++++++++++ examples/cli/testScenarios.yaml | 21 ++++++++++ examples/generate/policy_basic.yaml | 36 +++++++++++++++++ examples/mutate/overlay/output/nginx.yaml | 31 ++++++++++++++ .../mutate/overlay/{ => resources}/nginx.yaml | 2 + examples/mutate/overlay/testScenarios.yaml | 10 +++++ examples/mutate/patches/output/endpoints.yaml | 13 ++++-- examples/mutate/patches/testScenarios.yaml | 8 +++- pkg/testutils/testbundle.go | 17 +++++--- pkg/testutils/testsuite.go | 9 ++--- pkg/testutils/testutils.go | 6 ++- pkg/testutils/testutils_test.go | 28 ++++++++++--- pkg/testutils/utils.go | 10 +---- 14 files changed, 230 insertions(+), 30 deletions(-) create mode 100644 examples/cli/output/ghost.yaml create mode 100644 examples/cli/output/nginx.yaml create mode 100644 examples/cli/testScenarios.yaml create mode 100644 examples/generate/policy_basic.yaml create mode 100644 examples/mutate/overlay/output/nginx.yaml rename examples/mutate/overlay/{ => resources}/nginx.yaml (89%) create mode 100644 examples/mutate/overlay/testScenarios.yaml diff --git a/examples/cli/output/ghost.yaml b/examples/cli/output/ghost.yaml new file mode 100644 index 0000000000..751db9ec4c --- /dev/null +++ b/examples/cli/output/ghost.yaml @@ -0,0 +1,40 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ghost + creationTimestamp: + labels: + app: nginx_is_mutated + cli: test + isMutated: 'true' + nirmata.io/application.name: ghost + nirmata.io/component: ghost + nirmata.io/deployment.name: ghost +spec: + replicas: 1 + selector: + matchLabels: + nirmata.io/application.name: ghost + nirmata.io/component: ghost + template: + metadata: + creationTimestamp: + labels: + nirmata.io/application.name: ghost + nirmata.io/component: ghost + nirmata.io/deployment.name: ghost + spec: + containers: + - name: ghost + image: nginx:latest + ports: + - containerPort: 8080 + protocol: TCP + resources: {} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + revisionHistoryLimit: 5 +status: {} diff --git a/examples/cli/output/nginx.yaml b/examples/cli/output/nginx.yaml new file mode 100644 index 0000000000..9f4c121341 --- /dev/null +++ b/examples/cli/output/nginx.yaml @@ -0,0 +1,29 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: nginx-deployment + creationTimestamp: + labels: + app: nginx_is_mutated + cli: test + isMutated: 'true' +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + creationTimestamp: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.7.9 + ports: + - containerPort: 80 + resources: {} + imagePullPolicy: Always + strategy: {} +status: {} diff --git a/examples/cli/testScenarios.yaml b/examples/cli/testScenarios.yaml new file mode 100644 index 0000000000..158c4418bd --- /dev/null +++ b/examples/cli/testScenarios.yaml @@ -0,0 +1,21 @@ +# input +policy: policy-deployment +resource: nginx-deployment +initResources: +# expected +mutation: + mPatchedResource: nginx-deployment + reason: Success +validation: + reason: Success +--- +# input +policy: policy-deployment +resource: ghost +initResources: +# expected +mutation: + mPatchedResource: ghost + reason: Success +validation: + reason: Failed \ No newline at end of file diff --git a/examples/generate/policy_basic.yaml b/examples/generate/policy_basic.yaml new file mode 100644 index 0000000000..79f92ef081 --- /dev/null +++ b/examples/generate/policy_basic.yaml @@ -0,0 +1,36 @@ +apiVersion : kyverno.io/v1alpha1 +kind : Policy +metadata : + name : basic-policy +spec : + rules: + - name: "Basic config generator for all namespaces" + resource: + kinds: + - Namespace + selector: + matchLabels: + LabelForSelector : "namespace2" + generate: + kind: ConfigMap + name: default-config + clone: + namespace: default + name: config-template + - name: "Basic config generator for all namespaces" + resource: + kinds: + - Namespace + selector: + matchLabels: + LabelForSelector : "namespace2" + generate: + kind: Secret + name: mongo-creds + data: + data: + DB_USER: YWJyYWthZGFicmE= + DB_PASSWORD: YXBwc3dvcmQ= + metadata: + labels: + purpose: mongo diff --git a/examples/mutate/overlay/output/nginx.yaml b/examples/mutate/overlay/output/nginx.yaml new file mode 100644 index 0000000000..d59519c5a7 --- /dev/null +++ b/examples/mutate/overlay/output/nginx.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + creationTimestamp: + labels: + app: nginx +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + creationTimestamp: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 + resources: {} + imagePullPolicy: Always + - name: ghost + image: ghost:latest + resources: {} + imagePullPolicy: Always + strategy: {} +status: {} diff --git a/examples/mutate/overlay/nginx.yaml b/examples/mutate/overlay/resources/nginx.yaml similarity index 89% rename from examples/mutate/overlay/nginx.yaml rename to examples/mutate/overlay/resources/nginx.yaml index 20d0713896..bdf22b13cb 100644 --- a/examples/mutate/overlay/nginx.yaml +++ b/examples/mutate/overlay/resources/nginx.yaml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: nginx-deployment + creationTimestamp: labels: app: nginx spec: @@ -11,6 +12,7 @@ spec: app: nginx template: metadata: + creationTimestamp: labels: app: nginx spec: diff --git a/examples/mutate/overlay/testScenarios.yaml b/examples/mutate/overlay/testScenarios.yaml new file mode 100644 index 0000000000..e4012f24bb --- /dev/null +++ b/examples/mutate/overlay/testScenarios.yaml @@ -0,0 +1,10 @@ +# input +policy: set-image-pull-policy +resource: nginx-deployment +initResources: +# expected +mutation: + mPatchedResource: nginx-deployment + reason: Success +validation: + reason: Success \ No newline at end of file diff --git a/examples/mutate/patches/output/endpoints.yaml b/examples/mutate/patches/output/endpoints.yaml index 958d931482..c7ebca73a0 100644 --- a/examples/mutate/patches/output/endpoints.yaml +++ b/examples/mutate/patches/output/endpoints.yaml @@ -1,13 +1,20 @@ -apiVersion: v1 kind: Endpoints +apiVersion: v1 metadata: name: test-endpoint + creationTimestamp: labels: - label : test + label: test subsets: +- addresses: + - ip: 192.168.10.172 + ports: + - name: load-balancer-connection + port: 80 + protocol: UDP - addresses: - ip: 192.168.10.171 ports: - name: secure-connection - port: 443 + port: 9663 protocol: TCP diff --git a/examples/mutate/patches/testScenarios.yaml b/examples/mutate/patches/testScenarios.yaml index 7f0fec053a..df633e58df 100644 --- a/examples/mutate/patches/testScenarios.yaml +++ b/examples/mutate/patches/testScenarios.yaml @@ -9,7 +9,13 @@ mutation: validation: reason: Success --- +# input policy: policy-endpoints resource: test-endpoint initResources: -mPatchedResource: test-endpoint \ No newline at end of file +# expected +mutation: + mPatchedResource: test-endpoint + reason: Success +validation: + reason: Success \ No newline at end of file diff --git a/pkg/testutils/testbundle.go b/pkg/testutils/testbundle.go index b407fa97a0..3d4299902f 100644 --- a/pkg/testutils/testbundle.go +++ b/pkg/testutils/testbundle.go @@ -90,13 +90,14 @@ func loadScenarios(tbPath string, file string) ([]*tScenario, error) { glog.Warningf("Error while decoding YAML object, err: %s", err) continue } - fmt.Println(s.Policy) - fmt.Println(s.Resource) ts = append(ts, s) } return ts, nil } + +// Load test structure folder func (tb *testBundle) load() error { + // scenario file defines the mapping of resources and policies scenarios, err := loadScenarios(tb.path, tScenarioFile) if err != nil { return err @@ -116,8 +117,7 @@ func (tb *testBundle) load() error { } tb.policies[policy.GetName()] = policy } - - // load resources + // load trigger resources loadResources(tb.path, tb.resources, resourcesFolder) // load output resources loadResources(tb.path, tb.output, outputFolder) @@ -144,8 +144,10 @@ type testBundle struct { } func (tb *testBundle) run(t *testing.T, testingapplyTest IApplyTest) { + glog.Infof("Start: test on test bundles %s", tb.path) // run each scenario for _, ts := range tb.scenarios { + fmt.Println(tb.path) // get policy p, ok := tb.policies[ts.Policy] if !ok { @@ -167,6 +169,7 @@ func (tb *testBundle) run(t *testing.T, testingapplyTest IApplyTest) { tb.checkMutationResult(t, ts.Mutation, mPatchedResource, mResult) tb.checkValidationResult(t, ts.Validation, vResult) } + glog.Infof("Done: test on test bundles %s", tb.path) } func (tb *testBundle) checkValidationResult(t *testing.T, expect *tValidation, vResult result.Result) { @@ -191,13 +194,17 @@ func (tb *testBundle) checkMutationResult(t *testing.T, expect *tMutation, pr *r return } // get expected patched resource - er, ok := tb.resources[expect.MPatchedResource] + er, ok := tb.output[expect.MPatchedResource] if !ok { glog.Warningf("Resource %s not found", expect.MPatchedResource) return } // compare patched resources if !checkMutationRPatches(pr, er) { + fmt.Printf("Expected Resource %s \n", string(er.rawResource)) + fmt.Printf("Patched Resource %s \n", string(pr.rawResource)) + + glog.Warningf("Expected resource %s ", string(pr.rawResource)) t.Error("Patched resources not as expected") } // compare result diff --git a/pkg/testutils/testsuite.go b/pkg/testutils/testsuite.go index 7240ace08e..6da009b0fa 100644 --- a/pkg/testutils/testsuite.go +++ b/pkg/testutils/testsuite.go @@ -1,7 +1,6 @@ package testutils import ( - "fmt" "os" "path/filepath" "testing" @@ -41,20 +40,20 @@ type testSuite struct { func (ts *testSuite) buildTestSuite() error { // loading test bundles for test suite - fmt.Println(ts.path) err := filepath.Walk(ts.path, func(path string, info os.FileInfo, err error) error { if info.IsDir() { + glog.Infof("searching for test files at %s", path) // check if there are resources dir and policies yaml tb := NewTestBundle(path) if tb != nil { - // load resources + // try to load the test folder structure err := tb.load() if err != nil { - // glog.Error(err) + glog.Warningf("no supported test structure avaialbe at path %s", path) return nil } + glog.Infof("loading test suite at path %s", path) ts.tb = append(ts.tb, tb) - // fmt.Println(path) } } return nil diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 1d6d104d17..41ce4be588 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -1,11 +1,12 @@ package testutils import ( - "fmt" "io/ioutil" ospath "path" "path/filepath" "testing" + + "github.com/golang/glog" ) // Load policy & resource files @@ -19,6 +20,7 @@ const outputFolder string = "output" //LoadTestSuite reads the resource, policy and scenario files func LoadTestSuite(t *testing.T, path string) *testSuite { + glog.Infof("loading test suites at %s", path) // gp := os.Getenv("GOPATH") // ap := ospath.Join(gp, "src/github.com/nirmata/kyverno") // build test suite @@ -26,6 +28,7 @@ func LoadTestSuite(t *testing.T, path string) *testSuite { // ts := NewTestSuite(t, ospath.Join(ap, examplesPath)) ts := NewTestSuite(t, path) ts.buildTestSuite() + glog.Infof("done loading test suite at %s", path) return ts } @@ -43,7 +46,6 @@ func getYAMLfiles(path string) (yamls []string) { return nil } for _, file := range fileInfo { - fmt.Println(filepath.Ext(file.Name())) if file.Name() == tScenarioFile { continue } diff --git a/pkg/testutils/testutils_test.go b/pkg/testutils/testutils_test.go index cd0645de4a..fb68597038 100644 --- a/pkg/testutils/testutils_test.go +++ b/pkg/testutils/testutils_test.go @@ -1,20 +1,36 @@ package testutils import ( - "fmt" "testing" + + "github.com/golang/glog" ) -func TestUtils(t *testing.T) { - file := "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/mutate/patches" - ts := LoadTestSuite(t, file) +func runTest(t *testing.T, path string) { + // Load test suites at specified path + ts := LoadTestSuite(t, path) + // policy application logic tp := &testPolicy{} ts.setApplyTest(tp) + // run the tests for each test bundle ts.runTests() if ts != nil { - fmt.Println("Done building the test bundles") + glog.Infof("Done running the test at %s", path) + } +} + +func TestExamples(t *testing.T) { + // folders := []string{ + // "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/mutate/patches", + // "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/mutate/overlay", + // "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/cli", + // } + folders := []string{ + "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples", + } + for _, folder := range folders { + runTest(t, folder) } - // run the tests against the policy engine } diff --git a/pkg/testutils/utils.go b/pkg/testutils/utils.go index ccf4de6e15..c8819db794 100644 --- a/pkg/testutils/utils.go +++ b/pkg/testutils/utils.go @@ -61,7 +61,7 @@ type resourceInfo struct { func (ri resourceInfo) isSame(other resourceInfo) bool { // compare gvk - if ri.gvk != other.gvk { + if *ri.gvk != *other.gvk { return false } // compare rawResource @@ -69,7 +69,7 @@ func (ri resourceInfo) isSame(other resourceInfo) bool { } func getResourceYAML(d []byte) { - fmt.Println(string(d)) + // fmt.Println(string(d)) // convert json to yaml // print the result for reference // can be used as a dry run the get the expected result @@ -154,20 +154,16 @@ func (tp *testPolicy) applyPolicy(policy *policytypes.Policy, resource *resource err := mResult.ToError() if err == nil && len(mPatches) != 0 { patchedResource, err = engine.ApplyPatches(resource.rawResource, mPatches) - fmt.Println(len(resource.rawResource)) - fmt.Println(len(patchedResource)) if err != nil { return nil, nil, nil, err } // Validate vResult = engine.Validate(*policy, patchedResource, *resource.gvk) - fmt.Println(len(patchedResource)) } // Generate if client == nil { glog.Warning("Client is required to test generate") } - fmt.Println(len(patchedResource)) // transform the patched Resource into resource Info _, ri := extractResourceRaw(patchedResource) @@ -213,8 +209,6 @@ func LoadScenarios(file string) ([]*tScenario, error) { glog.Warningf("Error while decoding YAML object, err: %s", err) continue } - fmt.Println(s.Policy) - fmt.Println(s.Resource) ts = append(ts, s) } return ts, nil From b68745e8062debc82169e3835ab3d882fae2d33d Mon Sep 17 00:00:00 2001 From: shivdudhani Date: Thu, 13 Jun 2019 13:41:13 -0700 Subject: [PATCH 04/10] update files to support testing --- examples/generate/resources/configMap.yaml | 20 +++++++++++++++ examples/generate/resources/namespace.yaml | 7 ++++++ examples/generate/testScenarios.yaml | 29 ++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 examples/generate/resources/configMap.yaml create mode 100644 examples/generate/resources/namespace.yaml create mode 100644 examples/generate/testScenarios.yaml diff --git a/examples/generate/resources/configMap.yaml b/examples/generate/resources/configMap.yaml new file mode 100644 index 0000000000..8524e95d4a --- /dev/null +++ b/examples/generate/resources/configMap.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: game-config + namespace: default + labels: + originalLabel : isHere +data: + ui.properties : | + color.good=green + color.bad=red + + game.properties : | + enemies=predators + lives=3 + + configmap.data: | + ns=default + labels=originalLabel + labelscount=1 diff --git a/examples/generate/resources/namespace.yaml b/examples/generate/resources/namespace.yaml new file mode 100644 index 0000000000..566684da82 --- /dev/null +++ b/examples/generate/resources/namespace.yaml @@ -0,0 +1,7 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: "ns2" + labels: + LabelForSelector : "namespace2" + \ No newline at end of file diff --git a/examples/generate/testScenarios.yaml b/examples/generate/testScenarios.yaml new file mode 100644 index 0000000000..68b6427595 --- /dev/null +++ b/examples/generate/testScenarios.yaml @@ -0,0 +1,29 @@ +# input +policy: basic-policy +resource: ns2 +initResources: +# expected +mutation: + reason: Success +validation: + reason: Success +--- +# input +policy: zk-kafka-address +resource: ns2 +initResources: +# expected +mutation: + reason: Success +validation: + reason: Failed +--- +# input +policy: default +resource: ns2 +initResources: +# expected +mutation: + reason: Success +validation: + reason: Failed \ No newline at end of file From a49fb8cdabf19bb66818b6f67668ee36cc7b54c5 Mon Sep 17 00:00:00 2001 From: shivdudhani Date: Thu, 13 Jun 2019 13:41:38 -0700 Subject: [PATCH 05/10] refactor files --- pkg/testutils/testutils.go | 57 -------------------------------------- pkg/testutils/utils.go | 50 +++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 57 deletions(-) delete mode 100644 pkg/testutils/testutils.go diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go deleted file mode 100644 index 41ce4be588..0000000000 --- a/pkg/testutils/testutils.go +++ /dev/null @@ -1,57 +0,0 @@ -package testutils - -import ( - "io/ioutil" - ospath "path" - "path/filepath" - "testing" - - "github.com/golang/glog" -) - -// Load policy & resource files -// engine pass the (policy, resource) -// check the expected response - -const examplesPath string = "examples" -const resourcesFolder string = "resources" -const tScenarioFile string = "testScenarios.yaml" -const outputFolder string = "output" - -//LoadTestSuite reads the resource, policy and scenario files -func LoadTestSuite(t *testing.T, path string) *testSuite { - glog.Infof("loading test suites at %s", path) - // gp := os.Getenv("GOPATH") - // ap := ospath.Join(gp, "src/github.com/nirmata/kyverno") - // build test suite - // each suite contains test bundles for test sceanrios - // ts := NewTestSuite(t, ospath.Join(ap, examplesPath)) - ts := NewTestSuite(t, path) - ts.buildTestSuite() - glog.Infof("done loading test suite at %s", path) - return ts -} - -func checkMutationRPatches(er *resourceInfo, pr *resourceInfo) bool { - if !er.isSame(*pr) { - getResourceYAML(pr.rawResource) - return false - } - return true -} - -func getYAMLfiles(path string) (yamls []string) { - fileInfo, err := ioutil.ReadDir(path) - if err != nil { - return nil - } - for _, file := range fileInfo { - if file.Name() == tScenarioFile { - continue - } - if filepath.Ext(file.Name()) == ".yml" || filepath.Ext(file.Name()) == ".yaml" { - yamls = append(yamls, ospath.Join(path, file.Name())) - } - } - return yamls -} diff --git a/pkg/testutils/utils.go b/pkg/testutils/utils.go index c8819db794..76b781afd6 100644 --- a/pkg/testutils/utils.go +++ b/pkg/testutils/utils.go @@ -6,6 +6,9 @@ import ( "fmt" "io/ioutil" "os" + ospath "path" + "path/filepath" + "testing" "gopkg.in/yaml.v2" @@ -213,3 +216,50 @@ func LoadScenarios(file string) ([]*tScenario, error) { } return ts, nil } + +// Load policy & resource files +// engine pass the (policy, resource) +// check the expected response + +const examplesPath string = "examples" +const resourcesFolder string = "resources" +const tScenarioFile string = "testScenarios.yaml" +const outputFolder string = "output" + +//LoadTestSuite reads the resource, policy and scenario files +func LoadTestSuite(t *testing.T, path string) *testSuite { + glog.Infof("loading test suites at %s", path) + // gp := os.Getenv("GOPATH") + // ap := ospath.Join(gp, "src/github.com/nirmata/kyverno") + // build test suite + // each suite contains test bundles for test sceanrios + // ts := NewTestSuite(t, ospath.Join(ap, examplesPath)) + ts := NewTestSuite(t, path) + ts.buildTestSuite() + glog.Infof("done loading test suite at %s", path) + return ts +} + +func checkMutationRPatches(er *resourceInfo, pr *resourceInfo) bool { + if !er.isSame(*pr) { + getResourceYAML(pr.rawResource) + return false + } + return true +} + +func getYAMLfiles(path string) (yamls []string) { + fileInfo, err := ioutil.ReadDir(path) + if err != nil { + return nil + } + for _, file := range fileInfo { + if file.Name() == tScenarioFile { + continue + } + if filepath.Ext(file.Name()) == ".yml" || filepath.Ext(file.Name()) == ".yaml" { + yamls = append(yamls, ospath.Join(path, file.Name())) + } + } + return yamls +} From 0089eb348ed33a87d39f6f7ecc74ba8afd28d050 Mon Sep 17 00:00:00 2001 From: shivdudhani Date: Fri, 14 Jun 2019 14:51:06 -0700 Subject: [PATCH 06/10] generate resources --- examples/generate/namespace.yaml | 7 -- .../configMap_default.yaml} | 2 +- examples/generate/testScenarios.yaml | 35 +++++--- pkg/controller/controller_test.go | 7 +- pkg/dclient/client_test.go | 13 +-- pkg/dclient/utils.go | 23 +++--- pkg/testutils/testbundle.go | 81 ++++++++++++++++++- pkg/testutils/testutils_test.go | 2 +- pkg/testutils/utils.go | 49 ++++++++++- 9 files changed, 178 insertions(+), 41 deletions(-) delete mode 100644 examples/generate/namespace.yaml rename examples/generate/{configMap.yaml => resources/configMap_default.yaml} (92%) diff --git a/examples/generate/namespace.yaml b/examples/generate/namespace.yaml deleted file mode 100644 index 566684da82..0000000000 --- a/examples/generate/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Namespace -apiVersion: v1 -metadata: - name: "ns2" - labels: - LabelForSelector : "namespace2" - \ No newline at end of file diff --git a/examples/generate/configMap.yaml b/examples/generate/resources/configMap_default.yaml similarity index 92% rename from examples/generate/configMap.yaml rename to examples/generate/resources/configMap_default.yaml index 8524e95d4a..6fd8cbe74c 100644 --- a/examples/generate/configMap.yaml +++ b/examples/generate/resources/configMap_default.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: game-config + name: config-template namespace: default labels: originalLabel : isHere diff --git a/examples/generate/testScenarios.yaml b/examples/generate/testScenarios.yaml index 68b6427595..7cd22052ed 100644 --- a/examples/generate/testScenarios.yaml +++ b/examples/generate/testScenarios.yaml @@ -2,28 +2,39 @@ policy: basic-policy resource: ns2 initResources: + - default/config-template # expected -mutation: - reason: Success -validation: - reason: Success +generation: + resource: + - name: default-config + namespace: ns2 + kind: ConfigMap + - name: mongo-creds + namespace: ns2 + kind: Secret --- # input policy: zk-kafka-address resource: ns2 initResources: + - default/game-config # expected -mutation: - reason: Success -validation: - reason: Failed +generation: + resource: + - name: copied-cm + namespace: ns2 + kind: ConfigMap + - name: zk-kafka-address + namespace: ns2 + kind: ConfigMap --- # input policy: default resource: ns2 initResources: # expected -mutation: - reason: Success -validation: - reason: Failed \ No newline at end of file +generation: + resource: + - name: deny-all-traffic + namespace: ns2 + kind: NetworkPolicy \ No newline at end of file diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 8a300f1683..ca2b0d766f 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -10,6 +10,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/sample-controller/pkg/signals" ) @@ -78,8 +79,10 @@ func (f *fixture) setupFixture() { if err != nil { f.t.Fatal(err) } - regresource := map[string]string{"kyverno.io/v1alpha1": "policys"} - fclient.SetDiscovery(client.NewFakeDiscoveryClient(regresource)) + regResources := []schema.GroupVersionResource{ + schema.GroupVersionResource{Group: "kyverno.io/", Version: "v1alpha1", Resource: "policys"}} + + fclient.SetDiscovery(client.NewFakeDiscoveryClient(regResources)) } func newPolicy(name string) *types.Policy { diff --git a/pkg/dclient/client_test.go b/pkg/dclient/client_test.go index 984c316203..517871b258 100644 --- a/pkg/dclient/client_test.go +++ b/pkg/dclient/client_test.go @@ -9,6 +9,7 @@ import ( meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" ) // GetResource @@ -43,10 +44,12 @@ func newUnstructuredWithSpec(apiVersion, kind, namespace, name string, spec map[ func TestClient(t *testing.T) { scheme := runtime.NewScheme() // init groupversion - regresource := map[string]string{"group/version": "thekinds", - "group2/version": "thekinds", - "v1": "namespaces", - "apps/v1": "deployments"} + regResource := []schema.GroupVersionResource{ + schema.GroupVersionResource{Group: "group", Version: "version", Resource: "thekinds"}, + schema.GroupVersionResource{Group: "group2", Version: "version", Resource: "thekinds"}, + schema.GroupVersionResource{Group: "", Version: "v1", Resource: "namespaces"}, + schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}, + } // init resources objects := []runtime.Object{newUnstructured("group/version", "TheKind", "ns-foo", "name-foo"), newUnstructured("group2/version", "TheKind", "ns-foo", "name2-foo"), @@ -63,7 +66,7 @@ func TestClient(t *testing.T) { } // set discovery Client - client.SetDiscovery(NewFakeDiscoveryClient(regresource)) + client.SetDiscovery(NewFakeDiscoveryClient(regResource)) // Get Resource res, err := client.GetResource("thekinds", "ns-foo", "name-foo") if err != nil { diff --git a/pkg/dclient/utils.go b/pkg/dclient/utils.go index 080f2739c6..6ecde83bb4 100644 --- a/pkg/dclient/utils.go +++ b/pkg/dclient/utils.go @@ -36,16 +36,21 @@ func NewMockClient(scheme *runtime.Scheme, objects ...runtime.Object) (*Client, } // NewFakeDiscoveryClient returns a fakediscovery client -func NewFakeDiscoveryClient(regResources map[string]string) *fakeDiscoveryClient { - registeredResources := make([]schema.GroupVersionResource, len(regResources)) - for groupVersion, resource := range regResources { - gv, err := schema.ParseGroupVersion(groupVersion) - if err != nil { - continue - } - registeredResources = append(registeredResources, gv.WithResource(resource)) +func NewFakeDiscoveryClient(registeredResouces []schema.GroupVersionResource) *fakeDiscoveryClient { + // Load some-preregistd resources + res := []schema.GroupVersionResource{ + schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, + schema.GroupVersionResource{Version: "v1", Resource: "endpoints"}, + schema.GroupVersionResource{Version: "v1", Resource: "namespaces"}, + schema.GroupVersionResource{Version: "v1", Resource: "resourcequotas"}, + schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, + schema.GroupVersionResource{Version: "v1", Resource: "serviceaccounts"}, + schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "daemonsets"}, + schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}, + schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "statefulsets"}, } - return &fakeDiscoveryClient{registeredResouces: registeredResources} + registeredResouces = append(registeredResouces, res...) + return &fakeDiscoveryClient{registeredResouces: registeredResouces} } type fakeDiscoveryClient struct { diff --git a/pkg/testutils/testbundle.go b/pkg/testutils/testbundle.go index 3d4299902f..d408b60f39 100644 --- a/pkg/testutils/testbundle.go +++ b/pkg/testutils/testbundle.go @@ -9,8 +9,14 @@ import ( "github.com/golang/glog" policytypes "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" + dclient "github.com/nirmata/kyverno/pkg/dclient" "github.com/nirmata/kyverno/pkg/result" "gopkg.in/yaml.v2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + kscheme "k8s.io/client-go/kubernetes/scheme" + + "k8s.io/apimachinery/pkg/runtime/schema" ) func NewTestBundle(path string) *testBundle { @@ -143,11 +149,57 @@ type testBundle struct { scenarios []*tScenario } +func (tb *testBundle) createClient(t *testing.T, resources []string) *dclient.Client { + scheme := runtime.NewScheme() + objects := []runtime.Object{} + // registered group versions + regResources := []schema.GroupVersionResource{} + for _, resource := range resources { + // get resources + r, ok := tb.resources[resource] + if !ok { + glog.Warningf("Resource %s not found", resource) + continue + } + // get group version resource + gv := schema.GroupVersion{Group: r.gvk.Group, Version: r.gvk.Version} + gvr := gv.WithResource(getResourceFromKind(r.gvk.Kind)) + regResources = append(regResources, gvr) + + decode := kscheme.Codecs.UniversalDeserializer().Decode + obj, _, err := decode([]byte(r.rawResource), nil, nil) + if err != nil { + glog.Warning("Unable to deocde") + continue + } + // create unstructured + rdata, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&obj) + if err != nil { + fmt.Println(err) + continue + } + unstr := unstructured.Unstructured{Object: rdata} + objects = append(objects, &unstr) + } + // new mock client + // Mock Client + c, err := dclient.NewMockClient(scheme, objects...) + if err != nil { + t.Fatal(err) + } + // set discovery Client + c.SetDiscovery(dclient.NewFakeDiscoveryClient(regResources)) + + return c +} + func (tb *testBundle) run(t *testing.T, testingapplyTest IApplyTest) { glog.Infof("Start: test on test bundles %s", tb.path) // run each scenario for _, ts := range tb.scenarios { - fmt.Println(tb.path) + // TODO create client only for generate + // If there are init resources defined then load them + c := tb.createClient(t, ts.InitResources) // get policy p, ok := tb.policies[ts.Policy] if !ok { @@ -161,17 +213,42 @@ func (tb *testBundle) run(t *testing.T, testingapplyTest IApplyTest) { continue } // TODO: handle generate - mPatchedResource, mResult, vResult, err := testingapplyTest.applyPolicy(p, r, nil) + if ts.Generation != nil { + // assuming its namespaces creation + decode := kscheme.Codecs.UniversalDeserializer().Decode + obj, _, err := decode([]byte(r.rawResource), nil, nil) + _, err = c.CreateResource(getResourceFromKind(r.gvk.Kind), "", obj) + if err != nil { + t.Fatalf("error while creating namespace %s", ts.Resource) + } + } + + mPatchedResource, mResult, vResult, err := testingapplyTest.applyPolicy(p, r, c) if err != nil { t.Fatal(err) } // check the expected scenario tb.checkMutationResult(t, ts.Mutation, mPatchedResource, mResult) tb.checkValidationResult(t, ts.Validation, vResult) + tb.checkGeneration(t, ts.Generation, c) } glog.Infof("Done: test on test bundles %s", tb.path) } +func (tb *testBundle) checkGeneration(t *testing.T, expect *tGeneration, c *dclient.Client) { + if expect == nil { + glog.Info("No Generatin check defined") + return + } + // iterate throught the expected resources and check if the client has them + for _, r := range expect.Resources { + _, err := c.GetResource(getResourceFromKind(r.Kind), r.Namespace, r.Name) + if err != nil { + t.Errorf("Resource %s/%s of kind %s not found", r.Namespace, r.Name, r.Kind) + } + } +} + func (tb *testBundle) checkValidationResult(t *testing.T, expect *tValidation, vResult result.Result) { if expect == nil { glog.Info("No Validation check defined") diff --git a/pkg/testutils/testutils_test.go b/pkg/testutils/testutils_test.go index fb68597038..e91e89edae 100644 --- a/pkg/testutils/testutils_test.go +++ b/pkg/testutils/testutils_test.go @@ -28,7 +28,7 @@ func TestExamples(t *testing.T) { // "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/cli", // } folders := []string{ - "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples", + "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/generate", } for _, folder := range folders { runTest(t, folder) diff --git a/pkg/testutils/utils.go b/pkg/testutils/utils.go index 76b781afd6..c02607d371 100644 --- a/pkg/testutils/utils.go +++ b/pkg/testutils/utils.go @@ -114,6 +114,24 @@ func extractResource(resource string) (map[string]*resourceInfo, error) { return resources, nil } +func ParseApiVersionFromObject(bytes []byte) string { + var objectJSON map[string]interface{} + json.Unmarshal(bytes, &objectJSON) + if apiVersion, ok := objectJSON["apiVersion"].(string); ok { + return apiVersion + } + return "" +} + +func ParseKindFromObject(bytes []byte) string { + var objectJSON map[string]interface{} + json.Unmarshal(bytes, &objectJSON) + if kind, ok := objectJSON["kind"].(string); ok { + return kind + } + return "" +} + //ParseNameFromObject extracts resource name from JSON obj func ParseNameFromObject(bytes []byte) string { var objectJSON map[string]interface{} @@ -164,8 +182,8 @@ func (tp *testPolicy) applyPolicy(policy *policytypes.Policy, resource *resource vResult = engine.Validate(*policy, patchedResource, *resource.gvk) } // Generate - if client == nil { - glog.Warning("Client is required to test generate") + if client != nil { + engine.Generate(client, *policy, resource.rawResource, *resource.gvk) } // transform the patched Resource into resource Info @@ -181,6 +199,17 @@ type tScenario struct { InitResources []string `yaml:"initResources,omitempty"` Mutation *tMutation `yaml:"mutation,omitempty"` Validation *tValidation `yaml:"validation,omitempty"` + Generation *tGeneration `yaml:"generation,omitempty"` +} + +type tGeneration struct { + Resources []tResource `yaml:"resource"` +} + +type tResource struct { + Name string `yaml:"name"` + Namespace string `yaml:"namespace,omitempty"` + Kind string `yaml:"kind"` } type tValidation struct { @@ -263,3 +292,19 @@ func getYAMLfiles(path string) (yamls []string) { } return yamls } + +var kindToResource = map[string]string{ + "ConfigMap": "configmaps", + "Endpoints": "endpoints", + "Namespace": "namespaces", + "Secret": "secrets", + "Deployment": "deployments", + "NetworkPolicy": "networkpolicies", +} + +func getResourceFromKind(kind string) string { + if resource, ok := kindToResource[kind]; ok { + return resource + } + return "" +} From 987d005b731783ff77a50f53373ca66409c884a9 Mon Sep 17 00:00:00 2001 From: shivdudhani Date: Fri, 14 Jun 2019 15:20:05 -0700 Subject: [PATCH 07/10] rearrange test --- pkg/testutils/testbundle.go | 10 +++---- pkg/testutils/testutils_test.go | 50 ++++++++++++++++++++++----------- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/pkg/testutils/testbundle.go b/pkg/testutils/testbundle.go index d408b60f39..32d3ae2549 100644 --- a/pkg/testutils/testbundle.go +++ b/pkg/testutils/testbundle.go @@ -185,7 +185,7 @@ func (tb *testBundle) createClient(t *testing.T, resources []string) *dclient.Cl // Mock Client c, err := dclient.NewMockClient(scheme, objects...) if err != nil { - t.Fatal(err) + t.Error(err) } // set discovery Client c.SetDiscovery(dclient.NewFakeDiscoveryClient(regResources)) @@ -219,13 +219,13 @@ func (tb *testBundle) run(t *testing.T, testingapplyTest IApplyTest) { obj, _, err := decode([]byte(r.rawResource), nil, nil) _, err = c.CreateResource(getResourceFromKind(r.gvk.Kind), "", obj) if err != nil { - t.Fatalf("error while creating namespace %s", ts.Resource) + t.Errorf("error while creating namespace %s", ts.Resource) } } mPatchedResource, mResult, vResult, err := testingapplyTest.applyPolicy(p, r, c) if err != nil { - t.Fatal(err) + t.Error(err) } // check the expected scenario tb.checkMutationResult(t, ts.Mutation, mPatchedResource, mResult) @@ -237,7 +237,7 @@ func (tb *testBundle) run(t *testing.T, testingapplyTest IApplyTest) { func (tb *testBundle) checkGeneration(t *testing.T, expect *tGeneration, c *dclient.Client) { if expect == nil { - glog.Info("No Generatin check defined") + glog.Info("No Generate check defined") return } // iterate throught the expected resources and check if the client has them @@ -254,7 +254,6 @@ func (tb *testBundle) checkValidationResult(t *testing.T, expect *tValidation, v glog.Info("No Validation check defined") return } - // compare result // compare reason if len(expect.Reason) > 0 && expect.Reason != vResult.GetReason().String() { t.Error("Reason not matching") @@ -284,7 +283,6 @@ func (tb *testBundle) checkMutationResult(t *testing.T, expect *tMutation, pr *r glog.Warningf("Expected resource %s ", string(pr.rawResource)) t.Error("Patched resources not as expected") } - // compare result // compare reason if len(expect.Reason) > 0 && expect.Reason != mResult.GetReason().String() { t.Error("Reason not matching") diff --git a/pkg/testutils/testutils_test.go b/pkg/testutils/testutils_test.go index e91e89edae..dc683b19ba 100644 --- a/pkg/testutils/testutils_test.go +++ b/pkg/testutils/testutils_test.go @@ -6,31 +6,49 @@ import ( "github.com/golang/glog" ) +// func TestExamples(t *testing.T) { +// folders := []string{ +// "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples", +// } +// testrunner(t, folders) +// } + +func TestGenerate(t *testing.T) { + folders := []string{ + "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/generate", + } + testrunner(t, folders) +} + +func TestMutateOverlay(t *testing.T) { + folders := []string{ + "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/mutate/overlay", + } + testrunner(t, folders) +} + +func TestMutatePatches(t *testing.T) { + folders := []string{ + "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/mutate/patches", + } + testrunner(t, folders) +} + +func testrunner(t *testing.T, folders []string) { + for _, folder := range folders { + runTest(t, folder) + } +} + func runTest(t *testing.T, path string) { // Load test suites at specified path ts := LoadTestSuite(t, path) - // policy application logic tp := &testPolicy{} ts.setApplyTest(tp) - // run the tests for each test bundle ts.runTests() if ts != nil { glog.Infof("Done running the test at %s", path) } } - -func TestExamples(t *testing.T) { - // folders := []string{ - // "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/mutate/patches", - // "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/mutate/overlay", - // "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/cli", - // } - folders := []string{ - "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/generate", - } - for _, folder := range folders { - runTest(t, folder) - } -} From 5e0b1670c4a9e254c8889d2ac308bcc940726bab Mon Sep 17 00:00:00 2001 From: shivdudhani Date: Fri, 14 Jun 2019 16:06:14 -0700 Subject: [PATCH 08/10] skip with under dev tag --- pkg/testutils/testutils_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/testutils/testutils_test.go b/pkg/testutils/testutils_test.go index dc683b19ba..5c5a659f0e 100644 --- a/pkg/testutils/testutils_test.go +++ b/pkg/testutils/testutils_test.go @@ -14,6 +14,7 @@ import ( // } func TestGenerate(t *testing.T) { + t.Skip("Under development") folders := []string{ "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/generate", } @@ -21,6 +22,7 @@ func TestGenerate(t *testing.T) { } func TestMutateOverlay(t *testing.T) { + t.Skip("Under development") folders := []string{ "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/mutate/overlay", } @@ -28,6 +30,7 @@ func TestMutateOverlay(t *testing.T) { } func TestMutatePatches(t *testing.T) { + t.Skip("Under development") folders := []string{ "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/mutate/patches", } From e1df4a0dd908180faac0cc0b3e4a0b56bff33521 Mon Sep 17 00:00:00 2001 From: shivdudhani Date: Mon, 17 Jun 2019 18:11:22 -0700 Subject: [PATCH 09/10] rework the framework --- examples/cli/{resources => }/ghost.yaml | 0 examples/cli/{resources => }/nginx.yaml | 0 .../generate/{resources => }/configMap.yaml | 0 .../{resources => }/configMap_default.yaml | 0 .../generate/{resources => }/namespace.yaml | 0 .../mutate/overlay/{resources => }/nginx.yaml | 0 .../patches/{resources => }/endpoints.yaml | 0 pkg/testrunner/test.go | 180 ++++++++++ pkg/testrunner/testcase.go | 178 ++++++++++ pkg/testrunner/testrunner.go | 98 ++++++ pkg/testrunner/testrunner_test.go | 7 + pkg/testrunner/utils.go | 123 +++++++ pkg/testutils/testbundle.go | 294 ----------------- pkg/testutils/testsuite.go | 65 ---- pkg/testutils/testutils_test.go | 57 ---- pkg/testutils/utils.go | 310 ------------------ test/output/cm_copied_cm.yaml | 16 + test/output/cm_default_config.yaml | 16 + test/output/cm_zk-kafka-address.yaml | 8 + {examples/cli => test}/output/ghost.yaml | 0 {examples/cli => test}/output/nginx.yaml | 0 test/output/np_deny-all-traffic.yaml | 13 + .../output/op_overlay_nginx.yaml | 0 .../output/op_patches_endpoints.yaml | 0 test/output/sc_mongo_cred.yaml | 8 + test/scenarios/cli.yaml | 20 ++ test/scenarios/generate.yaml | 30 ++ test/scenarios/mutate/overlay.yaml | 8 + test/scenarios/mutate/patches.yaml | 8 + 29 files changed, 713 insertions(+), 726 deletions(-) rename examples/cli/{resources => }/ghost.yaml (100%) rename examples/cli/{resources => }/nginx.yaml (100%) rename examples/generate/{resources => }/configMap.yaml (100%) rename examples/generate/{resources => }/configMap_default.yaml (100%) rename examples/generate/{resources => }/namespace.yaml (100%) rename examples/mutate/overlay/{resources => }/nginx.yaml (100%) rename examples/mutate/patches/{resources => }/endpoints.yaml (100%) create mode 100644 pkg/testrunner/test.go create mode 100644 pkg/testrunner/testcase.go create mode 100644 pkg/testrunner/testrunner.go create mode 100644 pkg/testrunner/testrunner_test.go create mode 100644 pkg/testrunner/utils.go delete mode 100644 pkg/testutils/testbundle.go delete mode 100644 pkg/testutils/testsuite.go delete mode 100644 pkg/testutils/testutils_test.go delete mode 100644 pkg/testutils/utils.go create mode 100644 test/output/cm_copied_cm.yaml create mode 100644 test/output/cm_default_config.yaml create mode 100644 test/output/cm_zk-kafka-address.yaml rename {examples/cli => test}/output/ghost.yaml (100%) rename {examples/cli => test}/output/nginx.yaml (100%) create mode 100644 test/output/np_deny-all-traffic.yaml rename examples/mutate/overlay/output/nginx.yaml => test/output/op_overlay_nginx.yaml (100%) rename examples/mutate/patches/output/endpoints.yaml => test/output/op_patches_endpoints.yaml (100%) create mode 100644 test/output/sc_mongo_cred.yaml create mode 100644 test/scenarios/cli.yaml create mode 100644 test/scenarios/generate.yaml create mode 100644 test/scenarios/mutate/overlay.yaml create mode 100644 test/scenarios/mutate/patches.yaml diff --git a/examples/cli/resources/ghost.yaml b/examples/cli/ghost.yaml similarity index 100% rename from examples/cli/resources/ghost.yaml rename to examples/cli/ghost.yaml diff --git a/examples/cli/resources/nginx.yaml b/examples/cli/nginx.yaml similarity index 100% rename from examples/cli/resources/nginx.yaml rename to examples/cli/nginx.yaml diff --git a/examples/generate/resources/configMap.yaml b/examples/generate/configMap.yaml similarity index 100% rename from examples/generate/resources/configMap.yaml rename to examples/generate/configMap.yaml diff --git a/examples/generate/resources/configMap_default.yaml b/examples/generate/configMap_default.yaml similarity index 100% rename from examples/generate/resources/configMap_default.yaml rename to examples/generate/configMap_default.yaml diff --git a/examples/generate/resources/namespace.yaml b/examples/generate/namespace.yaml similarity index 100% rename from examples/generate/resources/namespace.yaml rename to examples/generate/namespace.yaml diff --git a/examples/mutate/overlay/resources/nginx.yaml b/examples/mutate/overlay/nginx.yaml similarity index 100% rename from examples/mutate/overlay/resources/nginx.yaml rename to examples/mutate/overlay/nginx.yaml diff --git a/examples/mutate/patches/resources/endpoints.yaml b/examples/mutate/patches/endpoints.yaml similarity index 100% rename from examples/mutate/patches/resources/endpoints.yaml rename to examples/mutate/patches/endpoints.yaml diff --git a/pkg/testrunner/test.go b/pkg/testrunner/test.go new file mode 100644 index 0000000000..1d41cfb413 --- /dev/null +++ b/pkg/testrunner/test.go @@ -0,0 +1,180 @@ +package testrunner + +import ( + "fmt" + "testing" + + ospath "path" + + "github.com/golang/glog" + pt "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" + client "github.com/nirmata/kyverno/pkg/dclient" + "github.com/nirmata/kyverno/pkg/engine" + "github.com/nirmata/kyverno/pkg/result" + kscheme "k8s.io/client-go/kubernetes/scheme" +) + +type test struct { + ap string + t *testing.T + testCase *testCase + // input + policy *pt.Policy + tResource *resourceInfo + loadResources []*resourceInfo + // expected + genResources []*resourceInfo + patchedResource *resourceInfo +} + +func (t *test) run() { + var client *client.Client + var err error + //mock client is used if generate is defined + if t.testCase.Expected.Generation != nil { + // create mock client & load resources + client, err = createClient(t.loadResources) + if err != nil { + t.t.Errorf("Unable to create client. err %s", err) + } + // TODO: handle generate + // assuming its namespaces creation + decode := kscheme.Codecs.UniversalDeserializer().Decode + obj, _, err := decode([]byte(t.tResource.rawResource), nil, nil) + _, err = client.CreateResource(getResourceFromKind(t.tResource.gvk.Kind), "", obj) + if err != nil { + t.t.Errorf("error while creating namespace %s", err) + } + + } + // apply the policy engine + pr, mResult, vResult, err := t.applyPolicy(t.policy, t.tResource, client) + if err != nil { + t.t.Error(err) + return + } + // Expected Result + t.checkMutationResult(pr, mResult) + t.checkValidationResult(vResult) + t.checkGenerationResult(client) +} + +func (t *test) checkMutationResult(pr *resourceInfo, result result.Result) { + if t.testCase.Expected.Mutation == nil { + glog.Info("No Mutation check defined") + return + } + // patched resource + if !compareResource(pr, t.patchedResource) { + fmt.Printf("Expected Resource %s \n", string(t.patchedResource.rawResource)) + fmt.Printf("Patched Resource %s \n", string(pr.rawResource)) + glog.Warningf("Expected resource %s ", string(pr.rawResource)) + t.t.Error("Patched resources not as expected") + } + // reason + reason := t.testCase.Expected.Mutation.Reason + if len(reason) > 0 && result.GetReason().String() != reason { + t.t.Error("Reason not matching") + } +} + +func (t *test) checkValidationResult(result result.Result) { + if t.testCase.Expected.Validation == nil { + glog.Info("No Validation check defined") + return + } + // reason + reason := t.testCase.Expected.Validation.Reason + if len(reason) > 0 && result.GetReason().String() != reason { + t.t.Error("Reason not matching") + } +} + +func (t *test) checkGenerationResult(client *client.Client) { + if t.testCase.Expected.Generation == nil { + glog.Info("No Generate check defined") + return + } + if client == nil { + glog.Info("client needs to be configured") + } + // check if the expected resources are generated + for _, r := range t.genResources { + n := ParseNameFromObject(r.rawResource) + ns := ParseNamespaceFromObject(r.rawResource) + _, err := client.GetResource(getResourceFromKind(r.gvk.Kind), ns, n) + if err != nil { + t.t.Errorf("Resource %s/%s of kinf %s not found", ns, n, r.gvk.Kind) + } + // compare if the resources are same + //TODO: comapre []bytes vs unstrcutured resource + } +} + +func (t *test) applyPolicy(policy *pt.Policy, + tresource *resourceInfo, + client *client.Client) (*resourceInfo, result.Result, result.Result, error) { + // apply policy on the trigger resource + // Mutate + var vResult result.Result + var patchedResource []byte + mPatches, mResult := engine.Mutate(*policy, tresource.rawResource, *tresource.gvk) + // TODO: only validate if there are no errors in mutate, why? + err := mResult.ToError() + if err == nil && len(mPatches) != 0 { + patchedResource, err = engine.ApplyPatches(tresource.rawResource, mPatches) + if err != nil { + return nil, nil, nil, err + } + // Validate + vResult = engine.Validate(*policy, patchedResource, *tresource.gvk) + } + // Generate + if client != nil { + engine.Generate(client, *policy, tresource.rawResource, *tresource.gvk) + } + // transform the patched Resource into resource Info + ri, err := extractResourceRaw(patchedResource) + if err != nil { + return nil, nil, nil, err + } + // return the results + return ri, mResult, vResult, nil +} + +func NewTest(ap string, t *testing.T, tc *testCase) (*test, error) { + //---INPUT--- + p, err := tc.loadPolicy(ospath.Join(ap, tc.Input.Policy)) + if err != nil { + return nil, err + } + r, err := tc.loadTriggerResource(ap) + if err != nil { + return nil, err + } + + lr, err := tc.loadPreloadedResources(ap) + if err != nil { + return nil, err + } + + //---EXPECTED--- + pr, err := tc.loadPatchedResource(ap) + if err != nil { + return nil, err + } + gr, err := tc.loadGeneratedResources(ap) + if err != nil { + return nil, err + } + return &test{ + ap: ap, + t: t, + testCase: tc, + policy: p, + tResource: r, + loadResources: lr, + genResources: gr, + patchedResource: pr, + }, nil +} diff --git a/pkg/testrunner/testcase.go b/pkg/testrunner/testcase.go new file mode 100644 index 0000000000..1bbdf70291 --- /dev/null +++ b/pkg/testrunner/testcase.go @@ -0,0 +1,178 @@ +package testrunner + +import ( + "bytes" + "encoding/json" + "fmt" + ospath "path" + + "github.com/golang/glog" + pt "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + yaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes/scheme" +) + +//testCase defines the input and the expected result +// it stores the path to the files that are to be loaded +// for references +type testCase struct { + Input *tInput `yaml:"input"` + Expected *tExpected `yaml:"expected"` +} + +// load resources store the resources that are pre-requisite +// for the test case and are pre-loaded in the client before +/// test case in evaluated +type tInput struct { + Policy string `yaml:"policy"` + Resource string `yaml:"resource"` + LoadResources []string `yaml:"load_resources,omitempty"` +} + +type tExpected struct { + Mutation *tMutation `yaml:"mutation,omitempty"` + Validation *tValidation `yaml:"validation,omitempty"` + Generation *tGeneration `yaml:"generation,omitempty"` +} + +type tMutation struct { + Patched_Resource string `yaml:"patched_resource,omitempty"` + tResult +} + +type tValidation struct { + tResult +} + +type tGeneration struct { + Resources []string `yaml:"resources"` +} + +type tResult struct { + Reason string `yaml:"reason, omitempty"` +} + +func (tc *testCase) policyEngineTest() { + +} +func (tc *testCase) loadPreloadedResources(ap string) ([]*resourceInfo, error) { + return loadResources(ap, tc.Input.LoadResources...) + // return loadResources(ap, tc.Input.LoadResources...) +} + +func (tc *testCase) loadGeneratedResources(ap string) ([]*resourceInfo, error) { + if tc.Expected.Generation == nil { + return nil, nil + } + return loadResources(ap, tc.Expected.Generation.Resources...) +} + +func (tc *testCase) loadPatchedResource(ap string) (*resourceInfo, error) { + if tc.Expected.Mutation == nil { + return nil, nil + } + rs, err := loadResources(ap, tc.Expected.Mutation.Patched_Resource) + if err != nil { + return nil, err + } + if len(rs) != 1 { + glog.Warning("expects single resource mutation but multiple defined, will use first one") + } + return rs[0], nil + +} +func (tc *testCase) loadResources(files []string) ([]*resourceInfo, error) { + lr := []*resourceInfo{} + for _, r := range files { + rs, err := loadResources(r) + if err != nil { + // return as test case will be invalid if a resource cannot be loaded + return nil, err + } + lr = append(lr, rs...) + } + return lr, nil +} + +func (tc *testCase) loadTriggerResource(ap string) (*resourceInfo, error) { + rs, err := loadResources(ap, tc.Input.Resource) + if err != nil { + return nil, err + } + if len(rs) != 1 { + glog.Warning("expects single resource trigger but multiple defined, will use first one") + } + return rs[0], nil + +} + +// Loads a single policy +func (tc *testCase) loadPolicy(file string) (*pt.Policy, error) { + p := &pt.Policy{} + data, err := LoadFile(file) + if err != nil { + return nil, err + } + pBytes, err := yaml.ToJSON(data) + if err != nil { + return nil, err + } + if err := json.Unmarshal(pBytes, p); err != nil { + return nil, err + } + if p.TypeMeta.Kind != "Policy" { + return nil, fmt.Errorf("failed to parse policy") + } + return p, nil +} + +// loads multiple resources +func loadResources(ap string, args ...string) ([]*resourceInfo, error) { + ris := []*resourceInfo{} + for _, file := range args { + data, err := LoadFile(ospath.Join(ap, file)) + if err != nil { + return nil, err + } + dd := bytes.Split(data, []byte(defaultYamlSeparator)) + // resources seperated by yaml seperator + for _, d := range dd { + ri, err := extractResourceRaw(d) + if err != nil { + glog.Errorf("unable to load resource. err: %s ", err) + continue + } + ris = append(ris, ri) + } + } + return ris, nil +} + +func extractResourceRaw(d []byte) (*resourceInfo, error) { + // decode := scheme.Codecs.UniversalDeserializer().Decode + // obj, gvk, err := decode(d, nil, nil) + // if err != nil { + // return nil, err + // } + obj, gvk, err := extractResourceUnMarshalled(d) + // runtime.object to JSON + raw, err := json.Marshal(obj) + if err != nil { + return nil, err + } + return &resourceInfo{rawResource: raw, + gvk: gvk}, nil +} + +func extractResourceUnMarshalled(d []byte) (runtime.Object, *metav1.GroupVersionKind, error) { + decode := scheme.Codecs.UniversalDeserializer().Decode + obj, gvk, err := decode(d, nil, nil) + if err != nil { + return nil, nil, err + } + return obj, &metav1.GroupVersionKind{Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind}, nil +} diff --git a/pkg/testrunner/testrunner.go b/pkg/testrunner/testrunner.go new file mode 100644 index 0000000000..1c217d97b6 --- /dev/null +++ b/pkg/testrunner/testrunner.go @@ -0,0 +1,98 @@ +package testrunner + +import ( + "bytes" + "io/ioutil" + "path/filepath" + "testing" + + "os" + ospath "path" + + "github.com/golang/glog" + "gopkg.in/yaml.v2" +) + +func runner(t *testing.T, relpath string) { + gp := os.Getenv("GOPATH") + ap := ospath.Join(gp, projectPath) + // build load scenarios + path := ospath.Join(ap, relpath) + // Load the scenario files + scenarioFiles := getYAMLfiles(path) + for _, secenarioFile := range scenarioFiles { + sc := newScenario(t, ap, secenarioFile) + if err := sc.load(); err != nil { + t.Error(err) + return + } + // run test cases + sc.run() + } +} + +type scenario struct { + ap string + t *testing.T + path string + tcs []*testCase +} + +func newScenario(t *testing.T, ap string, path string) *scenario { + return &scenario{ + ap: ap, + t: t, + path: path, + } +} + +func getYAMLfiles(path string) (yamls []string) { + fileInfo, err := ioutil.ReadDir(path) + if err != nil { + return nil + } + for _, file := range fileInfo { + if filepath.Ext(file.Name()) == ".yml" || filepath.Ext(file.Name()) == ".yaml" { + yamls = append(yamls, ospath.Join(path, file.Name())) + } + } + return yamls +} +func (sc *scenario) load() error { + // read file + data, err := LoadFile(sc.path) + if err != nil { + return err + } + tcs := []*testCase{} + // load test cases seperated by '---' + // each test case defines an input & expected result + dd := bytes.Split(data, []byte(defaultYamlSeparator)) + for _, d := range dd { + tc := &testCase{} + err := yaml.Unmarshal([]byte(d), tc) + if err != nil { + glog.Warningf("Error while decoding YAML object, err: %s", err) + continue + } + tcs = append(tcs, tc) + } + sc.tcs = tcs + return nil +} + +func (sc *scenario) run() { + if len(sc.tcs) == 0 { + sc.t.Error("No test cases to load") + return + } + + for _, tc := range sc.tcs { + t, err := NewTest(sc.ap, sc.t, tc) + if err != nil { + sc.t.Error(err) + continue + } + t.run() + } +} diff --git a/pkg/testrunner/testrunner_test.go b/pkg/testrunner/testrunner_test.go new file mode 100644 index 0000000000..50d949effd --- /dev/null +++ b/pkg/testrunner/testrunner_test.go @@ -0,0 +1,7 @@ +package testrunner + +import "testing" + +func TestExamples(t *testing.T) { + runner(t, "/test/scenarios") +} diff --git a/pkg/testrunner/utils.go b/pkg/testrunner/utils.go new file mode 100644 index 0000000000..eadd4c3f42 --- /dev/null +++ b/pkg/testrunner/utils.go @@ -0,0 +1,123 @@ +package testrunner + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "os" + + "github.com/golang/glog" + client "github.com/nirmata/kyverno/pkg/dclient" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + kscheme "k8s.io/client-go/kubernetes/scheme" +) + +const ( + defaultYamlSeparator = "---" + projectPath = "src/github.com/nirmata/kyverno" +) + +// LoadFile loads file in byte buffer +func LoadFile(path string) ([]byte, error) { + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil, err + } + return ioutil.ReadFile(path) +} + +type resourceInfo struct { + rawResource []byte + gvk *metav1.GroupVersionKind +} + +func (ri resourceInfo) isSame(other resourceInfo) bool { + // compare gvk + if *ri.gvk != *other.gvk { + return false + } + // compare rawResource + return bytes.Equal(ri.rawResource, other.rawResource) +} + +// compare patched resources +func compareResource(er *resourceInfo, pr *resourceInfo) bool { + if !er.isSame(*pr) { + return false + } + return true +} + +func createClient(resources []*resourceInfo) (*client.Client, error) { + scheme := runtime.NewScheme() + objects := []runtime.Object{} + // registered group versions + regResources := []schema.GroupVersionResource{} + + for _, r := range resources { + // registered gvr + gv := schema.GroupVersion{Group: r.gvk.Group, Version: r.gvk.Version} + gvr := gv.WithResource(getResourceFromKind(r.gvk.Kind)) + regResources = append(regResources, gvr) + decode := kscheme.Codecs.UniversalDeserializer().Decode + obj, _, err := decode([]byte(r.rawResource), nil, nil) + rdata, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&obj) + if err != nil { + glog.Errorf("failed to load resource. err %s", err) + } + unstr := unstructured.Unstructured{Object: rdata} + objects = append(objects, &unstr) + } + // Mock Client + c, err := client.NewMockClient(scheme, objects...) + if err != nil { + return nil, err + } + c.SetDiscovery(client.NewFakeDiscoveryClient(regResources)) + + return c, nil +} + +var kindToResource = map[string]string{ + "ConfigMap": "configmaps", + "Endpoints": "endpoints", + "Namespace": "namespaces", + "Secret": "secrets", + "Deployment": "deployments", + "NetworkPolicy": "networkpolicies", +} + +func getResourceFromKind(kind string) string { + if resource, ok := kindToResource[kind]; ok { + return resource + } + return "" +} + +//ParseNameFromObject extracts resource name from JSON obj +func ParseNameFromObject(bytes []byte) string { + var objectJSON map[string]interface{} + json.Unmarshal(bytes, &objectJSON) + + meta := objectJSON["metadata"].(map[string]interface{}) + + if name, ok := meta["name"].(string); ok { + return name + } + return "" +} + +// ParseNamespaceFromObject extracts the namespace from the JSON obj +func ParseNamespaceFromObject(bytes []byte) string { + var objectJSON map[string]interface{} + json.Unmarshal(bytes, &objectJSON) + + meta := objectJSON["metadata"].(map[string]interface{}) + + if namespace, ok := meta["namespace"].(string); ok { + return namespace + } + return "" +} diff --git a/pkg/testutils/testbundle.go b/pkg/testutils/testbundle.go deleted file mode 100644 index 32d3ae2549..0000000000 --- a/pkg/testutils/testbundle.go +++ /dev/null @@ -1,294 +0,0 @@ -package testutils - -import ( - "bytes" - "fmt" - "os" - ospath "path" - "testing" - - "github.com/golang/glog" - policytypes "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" - dclient "github.com/nirmata/kyverno/pkg/dclient" - "github.com/nirmata/kyverno/pkg/result" - "gopkg.in/yaml.v2" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - kscheme "k8s.io/client-go/kubernetes/scheme" - - "k8s.io/apimachinery/pkg/runtime/schema" -) - -func NewTestBundle(path string) *testBundle { - return &testBundle{ - path: path, - policies: make(map[string]*policytypes.Policy), - resources: make(map[string]*resourceInfo), - output: make(map[string]*resourceInfo), - } -} - -func loadResources(tbPath string, rs map[string]*resourceInfo, file string) { - path := ospath.Join(tbPath, file) - _, err := os.Stat(path) - if os.IsNotExist(err) { - glog.Warningf("%s directory not present at %s", file, tbPath) - return - } - // Load the resources from the output folder - yamls := getYAMLfiles(path) - if len(yamls) == 0 { - glog.Warningf("No resource yaml found at path %s", path) - return - } - for _, r := range yamls { - resources, err := extractResource(r) - if err != nil { - glog.Errorf("unable to extract resource: %s", err) - } - mergeResources(rs, resources) - } -} - -func (tb *testBundle) loadOutput() { - // check if output folder is defined - opath := ospath.Join(tb.path, outputFolder) - _, err := os.Stat(opath) - if os.IsNotExist(err) { - glog.Warningf("Output directory not present at %s", tb.path) - return - } - // Load the resources from the output folder - oYAMLs := getYAMLfiles(opath) - if len(oYAMLs) == 0 { - glog.Warningf("No resource yaml found at path %s", opath) - return - } - - for _, r := range oYAMLs { - resources, err := extractResource(r) - if err != nil { - glog.Errorf("unable to extract resource: %s", err) - } - mergeResources(tb.output, resources) - } -} - -func loadScenarios(tbPath string, file string) ([]*tScenario, error) { - // check if scenario yaml is defined - spath := ospath.Join(tbPath, file) - _, err := os.Stat(spath) - if os.IsNotExist(err) { - return nil, fmt.Errorf("Scenario file %s not defined at %s", file, tbPath) - } - ts := []*tScenario{} - // read the file - data, err := loadFile(spath) - if err != nil { - glog.Warningf("Error while loading file: %v\n", err) - return nil, err - } - dd := bytes.Split(data, []byte(defaultYamlSeparator)) - for _, d := range dd { - s := &tScenario{} - err := yaml.Unmarshal([]byte(d), s) - if err != nil { - glog.Warningf("Error while decoding YAML object, err: %s", err) - continue - } - ts = append(ts, s) - } - return ts, nil -} - -// Load test structure folder -func (tb *testBundle) load() error { - // scenario file defines the mapping of resources and policies - scenarios, err := loadScenarios(tb.path, tScenarioFile) - if err != nil { - return err - } - tb.scenarios = scenarios - // check if there are any files - pYAMLs := getYAMLfiles(tb.path) - if len(pYAMLs) == 0 { - return fmt.Errorf("No policy yaml found at path %s", tb.path) - } - for _, p := range pYAMLs { - // extract policy - policy, err := extractPolicy(p) - if err != nil { - glog.Errorf("unable to extract policy: %s", err) - continue - } - tb.policies[policy.GetName()] = policy - } - // load trigger resources - loadResources(tb.path, tb.resources, resourcesFolder) - // load output resources - loadResources(tb.path, tb.output, outputFolder) - - return nil -} - -func mergeResources(rs map[string]*resourceInfo, other map[string]*resourceInfo) { - for k, v := range other { - if _, ok := rs[k]; ok { - glog.Infof("resource already defined %s ", k) - continue - } - rs[k] = v - } -} - -type testBundle struct { - path string - policies map[string]*policytypes.Policy - resources map[string]*resourceInfo - output map[string]*resourceInfo - scenarios []*tScenario -} - -func (tb *testBundle) createClient(t *testing.T, resources []string) *dclient.Client { - scheme := runtime.NewScheme() - objects := []runtime.Object{} - // registered group versions - regResources := []schema.GroupVersionResource{} - for _, resource := range resources { - // get resources - r, ok := tb.resources[resource] - if !ok { - glog.Warningf("Resource %s not found", resource) - continue - } - // get group version resource - gv := schema.GroupVersion{Group: r.gvk.Group, Version: r.gvk.Version} - gvr := gv.WithResource(getResourceFromKind(r.gvk.Kind)) - regResources = append(regResources, gvr) - - decode := kscheme.Codecs.UniversalDeserializer().Decode - obj, _, err := decode([]byte(r.rawResource), nil, nil) - if err != nil { - glog.Warning("Unable to deocde") - continue - } - // create unstructured - rdata, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&obj) - if err != nil { - fmt.Println(err) - continue - } - unstr := unstructured.Unstructured{Object: rdata} - objects = append(objects, &unstr) - } - // new mock client - // Mock Client - c, err := dclient.NewMockClient(scheme, objects...) - if err != nil { - t.Error(err) - } - // set discovery Client - c.SetDiscovery(dclient.NewFakeDiscoveryClient(regResources)) - - return c -} - -func (tb *testBundle) run(t *testing.T, testingapplyTest IApplyTest) { - glog.Infof("Start: test on test bundles %s", tb.path) - // run each scenario - for _, ts := range tb.scenarios { - // TODO create client only for generate - // If there are init resources defined then load them - c := tb.createClient(t, ts.InitResources) - // get policy - p, ok := tb.policies[ts.Policy] - if !ok { - glog.Warningf("Policy %s not found", ts.Policy) - continue - } - // get resources - r, ok := tb.resources[ts.Resource] - if !ok { - glog.Warningf("Resource %s not found", ts.Resource) - continue - } - // TODO: handle generate - if ts.Generation != nil { - // assuming its namespaces creation - decode := kscheme.Codecs.UniversalDeserializer().Decode - obj, _, err := decode([]byte(r.rawResource), nil, nil) - _, err = c.CreateResource(getResourceFromKind(r.gvk.Kind), "", obj) - if err != nil { - t.Errorf("error while creating namespace %s", ts.Resource) - } - } - - mPatchedResource, mResult, vResult, err := testingapplyTest.applyPolicy(p, r, c) - if err != nil { - t.Error(err) - } - // check the expected scenario - tb.checkMutationResult(t, ts.Mutation, mPatchedResource, mResult) - tb.checkValidationResult(t, ts.Validation, vResult) - tb.checkGeneration(t, ts.Generation, c) - } - glog.Infof("Done: test on test bundles %s", tb.path) -} - -func (tb *testBundle) checkGeneration(t *testing.T, expect *tGeneration, c *dclient.Client) { - if expect == nil { - glog.Info("No Generate check defined") - return - } - // iterate throught the expected resources and check if the client has them - for _, r := range expect.Resources { - _, err := c.GetResource(getResourceFromKind(r.Kind), r.Namespace, r.Name) - if err != nil { - t.Errorf("Resource %s/%s of kind %s not found", r.Namespace, r.Name, r.Kind) - } - } -} - -func (tb *testBundle) checkValidationResult(t *testing.T, expect *tValidation, vResult result.Result) { - if expect == nil { - glog.Info("No Validation check defined") - return - } - // compare reason - if len(expect.Reason) > 0 && expect.Reason != vResult.GetReason().String() { - t.Error("Reason not matching") - } - // compare message - if len(expect.Message) > 0 && expect.Message != vResult.String() { - t.Error(("Message not matching")) - } -} - -func (tb *testBundle) checkMutationResult(t *testing.T, expect *tMutation, pr *resourceInfo, mResult result.Result) { - if expect == nil { - glog.Info("No Mutation check defined") - return - } - // get expected patched resource - er, ok := tb.output[expect.MPatchedResource] - if !ok { - glog.Warningf("Resource %s not found", expect.MPatchedResource) - return - } - // compare patched resources - if !checkMutationRPatches(pr, er) { - fmt.Printf("Expected Resource %s \n", string(er.rawResource)) - fmt.Printf("Patched Resource %s \n", string(pr.rawResource)) - - glog.Warningf("Expected resource %s ", string(pr.rawResource)) - t.Error("Patched resources not as expected") - } - // compare reason - if len(expect.Reason) > 0 && expect.Reason != mResult.GetReason().String() { - t.Error("Reason not matching") - } - // compare message - if len(expect.Message) > 0 && expect.Message != mResult.String() { - t.Error(("Message not matching")) - } -} diff --git a/pkg/testutils/testsuite.go b/pkg/testutils/testsuite.go deleted file mode 100644 index 6da009b0fa..0000000000 --- a/pkg/testutils/testsuite.go +++ /dev/null @@ -1,65 +0,0 @@ -package testutils - -import ( - "os" - "path/filepath" - "testing" - - "github.com/golang/glog" -) - -//NewTestSuite returns new test suite -func NewTestSuite(t *testing.T, path string) *testSuite { - return &testSuite{ - t: t, - path: path, - tb: []*testBundle{}, - } -} -func (ts *testSuite) runTests() { - //TODO : make sure the implementation the interface is pointing to is not nil - if ts.applyTest == nil { - glog.Error("Apply Test set for the test suite") - return - } - // for each test bundle run the test scenario - for _, tb := range ts.tb { - tb.run(ts.t, ts.applyTest) - } -} -func (ts *testSuite) setApplyTest(applyTest IApplyTest) { - ts.applyTest = applyTest -} - -type testSuite struct { - t *testing.T - path string - tb []*testBundle - applyTest IApplyTest -} - -func (ts *testSuite) buildTestSuite() error { - // loading test bundles for test suite - err := filepath.Walk(ts.path, func(path string, info os.FileInfo, err error) error { - if info.IsDir() { - glog.Infof("searching for test files at %s", path) - // check if there are resources dir and policies yaml - tb := NewTestBundle(path) - if tb != nil { - // try to load the test folder structure - err := tb.load() - if err != nil { - glog.Warningf("no supported test structure avaialbe at path %s", path) - return nil - } - glog.Infof("loading test suite at path %s", path) - ts.tb = append(ts.tb, tb) - } - } - return nil - }) - if err != nil { - ts.t.Fatal(err) - } - return nil -} diff --git a/pkg/testutils/testutils_test.go b/pkg/testutils/testutils_test.go deleted file mode 100644 index 5c5a659f0e..0000000000 --- a/pkg/testutils/testutils_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package testutils - -import ( - "testing" - - "github.com/golang/glog" -) - -// func TestExamples(t *testing.T) { -// folders := []string{ -// "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples", -// } -// testrunner(t, folders) -// } - -func TestGenerate(t *testing.T) { - t.Skip("Under development") - folders := []string{ - "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/generate", - } - testrunner(t, folders) -} - -func TestMutateOverlay(t *testing.T) { - t.Skip("Under development") - folders := []string{ - "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/mutate/overlay", - } - testrunner(t, folders) -} - -func TestMutatePatches(t *testing.T) { - t.Skip("Under development") - folders := []string{ - "/Users/shiv/nirmata/code/go/src/github.com/nirmata/kyverno/examples/mutate/patches", - } - testrunner(t, folders) -} - -func testrunner(t *testing.T, folders []string) { - for _, folder := range folders { - runTest(t, folder) - } -} - -func runTest(t *testing.T, path string) { - // Load test suites at specified path - ts := LoadTestSuite(t, path) - // policy application logic - tp := &testPolicy{} - ts.setApplyTest(tp) - // run the tests for each test bundle - ts.runTests() - if ts != nil { - glog.Infof("Done running the test at %s", path) - } -} diff --git a/pkg/testutils/utils.go b/pkg/testutils/utils.go deleted file mode 100644 index c02607d371..0000000000 --- a/pkg/testutils/utils.go +++ /dev/null @@ -1,310 +0,0 @@ -package testutils - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "os" - ospath "path" - "path/filepath" - "testing" - - "gopkg.in/yaml.v2" - - "github.com/golang/glog" - policytypes "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" - client "github.com/nirmata/kyverno/pkg/dclient" - "github.com/nirmata/kyverno/pkg/engine" - "github.com/nirmata/kyverno/pkg/result" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kyaml "k8s.io/apimachinery/pkg/util/yaml" - "k8s.io/client-go/kubernetes/scheme" -) - -const ( - defaultYamlSeparator = "---" -) - -func loadFile(fileDir string) ([]byte, error) { - if _, err := os.Stat(fileDir); os.IsNotExist(err) { - return nil, err - } - return ioutil.ReadFile(fileDir) -} - -func extractPolicy(fileDir string) (*policytypes.Policy, error) { - policy := &policytypes.Policy{} - - file, err := loadFile(fileDir) - if err != nil { - return nil, fmt.Errorf("failed to load file: %v", err) - } - - policyBytes, err := kyaml.ToJSON(file) - if err != nil { - return nil, err - } - - if err := json.Unmarshal(policyBytes, policy); err != nil { - return nil, fmt.Errorf("failed to decode policy %s, err: %v", policy.Name, err) - } - - if policy.TypeMeta.Kind != "Policy" { - return nil, fmt.Errorf("failed to parse policy") - } - - return policy, nil -} - -type resourceInfo struct { - rawResource []byte - gvk *metav1.GroupVersionKind -} - -func (ri resourceInfo) isSame(other resourceInfo) bool { - // compare gvk - if *ri.gvk != *other.gvk { - return false - } - // compare rawResource - return bytes.Equal(ri.rawResource, other.rawResource) -} - -func getResourceYAML(d []byte) { - // fmt.Println(string(d)) - // convert json to yaml - // print the result for reference - // can be used as a dry run the get the expected result -} - -func extractResourceRaw(d []byte) (string, *resourceInfo) { - decode := scheme.Codecs.UniversalDeserializer().Decode - obj, gvk, err := decode([]byte(d), nil, nil) - if err != nil { - glog.Warningf("Error while decoding YAML object, err: %s\n", err) - return "", nil - } - raw, err := json.Marshal(obj) - if err != nil { - glog.Warningf("Error while marshalling manifest, err: %v\n", err) - return "", nil - } - gvkInfo := &metav1.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind} - rn := ParseNameFromObject(raw) - rns := ParseNamespaceFromObject(raw) - if rns != "" { - rn = rns + "/" + rn - } - return rn, &resourceInfo{rawResource: raw, gvk: gvkInfo} -} - -func extractResource(resource string) (map[string]*resourceInfo, error) { - resources := make(map[string]*resourceInfo) - data, err := loadFile(resource) - if err != nil { - glog.Warningf("Error while loading file: %v\n", err) - return nil, err - } - dd := bytes.Split(data, []byte(defaultYamlSeparator)) - for _, d := range dd { - rn, r := extractResourceRaw(d) - resources[rn] = r - } - return resources, nil -} - -func ParseApiVersionFromObject(bytes []byte) string { - var objectJSON map[string]interface{} - json.Unmarshal(bytes, &objectJSON) - if apiVersion, ok := objectJSON["apiVersion"].(string); ok { - return apiVersion - } - return "" -} - -func ParseKindFromObject(bytes []byte) string { - var objectJSON map[string]interface{} - json.Unmarshal(bytes, &objectJSON) - if kind, ok := objectJSON["kind"].(string); ok { - return kind - } - return "" -} - -//ParseNameFromObject extracts resource name from JSON obj -func ParseNameFromObject(bytes []byte) string { - var objectJSON map[string]interface{} - json.Unmarshal(bytes, &objectJSON) - - meta := objectJSON["metadata"].(map[string]interface{}) - - if name, ok := meta["name"].(string); ok { - return name - } - return "" -} - -// ParseNamespaceFromObject extracts the namespace from the JSON obj -func ParseNamespaceFromObject(bytes []byte) string { - var objectJSON map[string]interface{} - json.Unmarshal(bytes, &objectJSON) - - meta := objectJSON["metadata"].(map[string]interface{}) - - if namespace, ok := meta["namespace"].(string); ok { - return namespace - } - return "" -} - -type IApplyTest interface { - applyPolicy(policy *policytypes.Policy, resource *resourceInfo, client *client.Client) (*resourceInfo, result.Result, result.Result, error) -} - -type testPolicy struct { -} - -func (tp *testPolicy) applyPolicy(policy *policytypes.Policy, resource *resourceInfo, client *client.Client) (*resourceInfo, result.Result, result.Result, error) { - // apply policy on the trigger resource - // Mutate - var vResult result.Result - var patchedResource []byte - mPatches, mResult := engine.Mutate(*policy, resource.rawResource, *resource.gvk) - // TODO: only validate if there are no errors in mutate, why? - err := mResult.ToError() - if err == nil && len(mPatches) != 0 { - patchedResource, err = engine.ApplyPatches(resource.rawResource, mPatches) - if err != nil { - return nil, nil, nil, err - } - // Validate - vResult = engine.Validate(*policy, patchedResource, *resource.gvk) - } - // Generate - if client != nil { - engine.Generate(client, *policy, resource.rawResource, *resource.gvk) - } - - // transform the patched Resource into resource Info - _, ri := extractResourceRaw(patchedResource) - // return the results - return ri, mResult, vResult, nil - // TODO: merge the results for mutation and validation -} - -type tScenario struct { - Policy string `yaml:"policy"` - Resource string `yaml:"resource"` - InitResources []string `yaml:"initResources,omitempty"` - Mutation *tMutation `yaml:"mutation,omitempty"` - Validation *tValidation `yaml:"validation,omitempty"` - Generation *tGeneration `yaml:"generation,omitempty"` -} - -type tGeneration struct { - Resources []tResource `yaml:"resource"` -} - -type tResource struct { - Name string `yaml:"name"` - Namespace string `yaml:"namespace,omitempty"` - Kind string `yaml:"kind"` -} - -type tValidation struct { - Reason string `yaml:"reason,omitempty"` - Message string `yaml:"message,omitempty"` - Error string `yaml:"error,omitempty"` -} - -type tMutation struct { - MPatchedResource string `yaml:"mPatchedResource,omitempty"` - Reason string `yaml:"reason,omitempty"` - Message string `yaml:"message,omitempty"` - Error string `yaml:"error,omitempty"` -} - -func LoadScenarios(file string) ([]*tScenario, error) { - ts := []*tScenario{} - // read the file - data, err := loadFile(file) - if err != nil { - glog.Warningf("Error while loading file: %v\n", err) - return nil, err - } - dd := bytes.Split(data, []byte(defaultYamlSeparator)) - for _, d := range dd { - s := &tScenario{} - err := yaml.Unmarshal([]byte(d), s) - if err != nil { - glog.Warningf("Error while decoding YAML object, err: %s", err) - continue - } - ts = append(ts, s) - } - return ts, nil -} - -// Load policy & resource files -// engine pass the (policy, resource) -// check the expected response - -const examplesPath string = "examples" -const resourcesFolder string = "resources" -const tScenarioFile string = "testScenarios.yaml" -const outputFolder string = "output" - -//LoadTestSuite reads the resource, policy and scenario files -func LoadTestSuite(t *testing.T, path string) *testSuite { - glog.Infof("loading test suites at %s", path) - // gp := os.Getenv("GOPATH") - // ap := ospath.Join(gp, "src/github.com/nirmata/kyverno") - // build test suite - // each suite contains test bundles for test sceanrios - // ts := NewTestSuite(t, ospath.Join(ap, examplesPath)) - ts := NewTestSuite(t, path) - ts.buildTestSuite() - glog.Infof("done loading test suite at %s", path) - return ts -} - -func checkMutationRPatches(er *resourceInfo, pr *resourceInfo) bool { - if !er.isSame(*pr) { - getResourceYAML(pr.rawResource) - return false - } - return true -} - -func getYAMLfiles(path string) (yamls []string) { - fileInfo, err := ioutil.ReadDir(path) - if err != nil { - return nil - } - for _, file := range fileInfo { - if file.Name() == tScenarioFile { - continue - } - if filepath.Ext(file.Name()) == ".yml" || filepath.Ext(file.Name()) == ".yaml" { - yamls = append(yamls, ospath.Join(path, file.Name())) - } - } - return yamls -} - -var kindToResource = map[string]string{ - "ConfigMap": "configmaps", - "Endpoints": "endpoints", - "Namespace": "namespaces", - "Secret": "secrets", - "Deployment": "deployments", - "NetworkPolicy": "networkpolicies", -} - -func getResourceFromKind(kind string) string { - if resource, ok := kindToResource[kind]; ok { - return resource - } - return "" -} diff --git a/test/output/cm_copied_cm.yaml b/test/output/cm_copied_cm.yaml new file mode 100644 index 0000000000..e1ffd57b59 --- /dev/null +++ b/test/output/cm_copied_cm.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +data: + configmap.data: | + ns=default + labels=originalLabel + labelscount=1 + game.properties: | + enemies=predators + lives=3 + ui.properties: "color.good=green\ncolor.bad=red \n" +kind: ConfigMap +metadata: + labels: + originalLabel: isHere + name: copied-cm + namespace: ns2 \ No newline at end of file diff --git a/test/output/cm_default_config.yaml b/test/output/cm_default_config.yaml new file mode 100644 index 0000000000..08ef79a116 --- /dev/null +++ b/test/output/cm_default_config.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +data: + configmap.data: | + ns=default + labels=originalLabel + labelscount=1 + game.properties: | + enemies=predators + lives=3 + ui.properties: "color.good=green\ncolor.bad=red \n" +kind: ConfigMap +metadata: + labels: + originalLabel: isHere + name: default-config + namespace: ns2 \ No newline at end of file diff --git a/test/output/cm_zk-kafka-address.yaml b/test/output/cm_zk-kafka-address.yaml new file mode 100644 index 0000000000..18aaecaa3c --- /dev/null +++ b/test/output/cm_zk-kafka-address.yaml @@ -0,0 +1,8 @@ +data: + KAFKA_ADDRESS: 192.168.10.13:9092,192.168.10.14:9092,192.168.10.15:9092 + ZK_ADDRESS: 192.168.10.10:2181,192.168.10.11:2181,192.168.10.12:2181 +kind: ConfigMap +apiVersion: v1 +metadata: + name: zk-kafka-address + namespace: ns2 diff --git a/examples/cli/output/ghost.yaml b/test/output/ghost.yaml similarity index 100% rename from examples/cli/output/ghost.yaml rename to test/output/ghost.yaml diff --git a/examples/cli/output/nginx.yaml b/test/output/nginx.yaml similarity index 100% rename from examples/cli/output/nginx.yaml rename to test/output/nginx.yaml diff --git a/test/output/np_deny-all-traffic.yaml b/test/output/np_deny-all-traffic.yaml new file mode 100644 index 0000000000..dfe3faab98 --- /dev/null +++ b/test/output/np_deny-all-traffic.yaml @@ -0,0 +1,13 @@ +metadata: + annotations: {} + labels: + policyname: default + name: deny-all-traffic + namespace: ns2 +podSelector: + matchExpressions: [] + matchLabels: {} +policyTypes: [] +spec: +kind: NetworkPolicy +apiVersion: extensions/v1beta1 \ No newline at end of file diff --git a/examples/mutate/overlay/output/nginx.yaml b/test/output/op_overlay_nginx.yaml similarity index 100% rename from examples/mutate/overlay/output/nginx.yaml rename to test/output/op_overlay_nginx.yaml diff --git a/examples/mutate/patches/output/endpoints.yaml b/test/output/op_patches_endpoints.yaml similarity index 100% rename from examples/mutate/patches/output/endpoints.yaml rename to test/output/op_patches_endpoints.yaml diff --git a/test/output/sc_mongo_cred.yaml b/test/output/sc_mongo_cred.yaml new file mode 100644 index 0000000000..cf004898a8 --- /dev/null +++ b/test/output/sc_mongo_cred.yaml @@ -0,0 +1,8 @@ +data: + DB_PASSWORD: YXBwc3dvcmQ= + DB_USER: YWJyYWthZGFicmE= +metadata: + name: mongo-creds + namespace: ns2 +kind: Secret +apiVersion: v1 \ No newline at end of file diff --git a/test/scenarios/cli.yaml b/test/scenarios/cli.yaml new file mode 100644 index 0000000000..1e9496b983 --- /dev/null +++ b/test/scenarios/cli.yaml @@ -0,0 +1,20 @@ +# file path relative to project root +input: + policy: examples/cli/policy_deployment.yaml + resource: examples/cli/nginx.yaml +expected: + mutation: + patched_resource: test/output/nginx.yaml + reason: Success + validation: + reason: Success +--- +input: + policy: examples/cli/policy_deployment.yaml + resource: examples/cli/ghost.yaml +expected: + mutation: + patched_resource: test/output/ghost.yaml + reason: Success + validation: + reason: Success \ No newline at end of file diff --git a/test/scenarios/generate.yaml b/test/scenarios/generate.yaml new file mode 100644 index 0000000000..b305067a04 --- /dev/null +++ b/test/scenarios/generate.yaml @@ -0,0 +1,30 @@ +# file path relative to project root +input: + policy: examples/generate/policy_basic.yaml + resource: examples/generate/namespace.yaml + load_resources: + - examples/generate/configMap_default.yaml +expected: + generation: + resources: + - test/output/cm_default_config.yaml + - test/output/sc_mongo_cred.yaml +--- +input: + policy: examples/generate/policy_generate.yaml + resource: examples/generate/namespace.yaml + load_resources: + - examples/generate/configMap.yaml +expected: + generation: + resources: + - test/output/cm_copied_cm.yaml + - test/output/cm_zk-kafka-address.yaml +--- +input: + policy: examples/generate/policy_networkPolicy.yaml + resource: examples/generate/namespace.yaml +expected: + generation: + resources: + - test/output/np_deny-all-traffic.yaml \ No newline at end of file diff --git a/test/scenarios/mutate/overlay.yaml b/test/scenarios/mutate/overlay.yaml new file mode 100644 index 0000000000..a2db4326c5 --- /dev/null +++ b/test/scenarios/mutate/overlay.yaml @@ -0,0 +1,8 @@ +# file path relative to project root +input: + policy: examples/mutate/overlay/policy_imagePullPolicy.yaml + resource: examples/mutate/overlay/nginx.yaml +expected: + mutation: + patched_resource: test/output/op_overlay_nginx.yaml + reason: Success \ No newline at end of file diff --git a/test/scenarios/mutate/patches.yaml b/test/scenarios/mutate/patches.yaml new file mode 100644 index 0000000000..70de56af2f --- /dev/null +++ b/test/scenarios/mutate/patches.yaml @@ -0,0 +1,8 @@ +# file path relative to project root +input: + policy: examples/mutate/patches/policy_endpoints.yaml + resource: examples/mutate/patches/endpoints.yaml +expected: + mutation: + patched_resource: test/output/op_patches_endpoints.yaml + reason: Success \ No newline at end of file From 40e849836307739b2e18b2ec934980d3ff750b4a Mon Sep 17 00:00:00 2001 From: shivdudhani Date: Wed, 19 Jun 2019 15:12:48 -0700 Subject: [PATCH 10/10] rebase with master --- pkg/testrunner/test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/testrunner/test.go b/pkg/testrunner/test.go index 1d41cfb413..f475f2e579 100644 --- a/pkg/testrunner/test.go +++ b/pkg/testrunner/test.go @@ -41,7 +41,7 @@ func (t *test) run() { // assuming its namespaces creation decode := kscheme.Codecs.UniversalDeserializer().Decode obj, _, err := decode([]byte(t.tResource.rawResource), nil, nil) - _, err = client.CreateResource(getResourceFromKind(t.tResource.gvk.Kind), "", obj) + _, err = client.CreateResource(getResourceFromKind(t.tResource.gvk.Kind), "", obj, false) if err != nil { t.t.Errorf("error while creating namespace %s", err) }