From 087b4330d45900fd1a386fc88ad5b80cd0ded57c Mon Sep 17 00:00:00 2001 From: NoSkillGirl Date: Mon, 21 Jun 2021 20:12:14 +0530 Subject: [PATCH] added metrics check for policy creation Signed-off-by: NoSkillGirl --- pkg/config/config.go | 6 -- pkg/webhooks/server.go | 11 --- test/e2e/common/common.go | 20 +++++ test/e2e/generate/config.go | 13 ++++ test/e2e/generate/generate_test.go | 114 ++++++++++++++++++++++++----- test/e2e/metrics/metrics_test.go | 63 ---------------- test/e2e/mutate/mutate_test.go | 35 +++++++++ 7 files changed, 162 insertions(+), 100 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 4193a6e096..056dfdb396 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -78,12 +78,6 @@ var ( //VerifyMutatingWebhookServicePath is the path for verify webhook(used to veryfing if admission control is enabled and active) VerifyMutatingWebhookServicePath = "/verifymutate" - // GetCachePath is the path for getting latest cache - GetCachePath = "/cache" - - // CacheSyncPath is the path for syncing the cache - CacheSyncPath = "/cache/sync" - // LivenessServicePath is the path for check liveness health LivenessServicePath = "/health/liveness" diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index ba6c35eaac..f2c161d1fa 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -215,17 +215,6 @@ func NewWebhookServer( mux.HandlerFunc("POST", config.PolicyValidatingWebhookServicePath, ws.handlerFunc(ws.policyValidation, true)) mux.HandlerFunc("POST", config.VerifyMutatingWebhookServicePath, ws.handlerFunc(ws.verifyHandler, false)) - mux.HandlerFunc("GET", config.GetCachePath, func(w http.ResponseWriter, r *http.Request) { - logger := ws.log.WithValues("action", "get cache") - defer r.Body.Close() - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - err := json.NewEncoder(w).Encode(pCache) - if err != nil { - logger.Error(err, "error in the JSON encoding") - } - }) - // Handle Liveness responds to a Kubernetes Liveness probe // Fail this request if Kubernetes should restart this instance mux.HandlerFunc("GET", config.LivenessServicePath, func(w http.ResponseWriter, r *http.Request) { diff --git a/test/e2e/common/common.go b/test/e2e/common/common.go index c21f3b8856..bf283b597c 100644 --- a/test/e2e/common/common.go +++ b/test/e2e/common/common.go @@ -1,11 +1,31 @@ package common import ( + "bytes" "fmt" "strings" "time" + + "github.com/kyverno/kyverno/test/e2e" ) +func CallMetrics() (string, error) { + requestObj := e2e.APIRequest{ + URL: "http://localhost:8000/metrics", + Type: "GET", + } + + response, err := e2e.CallAPI(requestObj) + if err != nil { + return "", err + } + + buf := new(bytes.Buffer) + buf.ReadFrom(response.Body) + newStr := buf.String() + return newStr, nil +} + func ProcessMetrics(newStr, e2ePolicyName string, e2eTime time.Time) bool { fmt.Println("e2ePolicyName: ", e2ePolicyName, "e2eTime: ", e2eTime) var action, policyName string diff --git a/test/e2e/generate/config.go b/test/e2e/generate/config.go index b030de2663..354a658e4a 100644 --- a/test/e2e/generate/config.go +++ b/test/e2e/generate/config.go @@ -23,6 +23,8 @@ var RoleTests = []struct { Sync bool // Data - The Yaml file of the ClusterPolicy of the ROle and RoleBinding - ([]byte{}) Data []byte + // PolicyName - Name of the Policy + PolicyName string }{ { TestName: "test-role-rolebinding-without-clone", @@ -32,6 +34,7 @@ var RoleTests = []struct { Clone: false, Sync: false, Data: roleRoleBindingYamlWithSync, + PolicyName: "gen-role-policy", }, { TestName: "test-role-rolebinding-withsync-without-clone", @@ -41,6 +44,7 @@ var RoleTests = []struct { Clone: false, Sync: true, Data: roleRoleBindingYamlWithSync, + PolicyName: "gen-role-policy", }, { TestName: "test-role-rolebinding-with-clone", @@ -53,6 +57,7 @@ var RoleTests = []struct { CloneNamespace: "default", Sync: false, Data: roleRoleBindingYamlWithClone, + PolicyName: "gen-role", }, } @@ -82,6 +87,8 @@ var ClusterRoleTests = []struct { Sync bool // Data - The Yaml file of the ClusterPolicy of the ClusterRole and ClusterRoleBinding - ([]byte{}) Data []byte + // PolicyName - Name of the Policy + PolicyName string }{ { TestName: "test-clusterrole-clusterrolebinding-without-clone", @@ -91,6 +98,7 @@ var ClusterRoleTests = []struct { Clone: false, Sync: false, Data: genClusterRoleYamlWithSync, + PolicyName: "gen-cluster-policy", }, { TestName: "test-clusterrole-clusterrolebinding-with-sync-without-clone", @@ -100,6 +108,7 @@ var ClusterRoleTests = []struct { Clone: false, Sync: true, Data: genClusterRoleYamlWithSync, + PolicyName: "gen-cluster-policy", }, { TestName: "test-clusterrole-clusterrolebinding-with-sync-with-clone", @@ -113,6 +122,7 @@ var ClusterRoleTests = []struct { CloneSourceClusterRoleBindingData: baseClusterRoleBindingData, Sync: false, Data: genClusterRoleYamlWithSync, + PolicyName: "gen-cluster-policy", }, } @@ -254,6 +264,8 @@ var SourceResourceUpdateReplicationTests = []struct { ConfigMapName string // CloneSourceConfigMapData - Source ConfigMap Yaml CloneSourceConfigMapData []byte + // PolicyName - Name of the Policy + PolicyName string }{ { TestName: "test-clone-source-resource-update-replication", @@ -264,5 +276,6 @@ var SourceResourceUpdateReplicationTests = []struct { ConfigMapName: "game-demo", CloneNamespace: "default", CloneSourceConfigMapData: cloneSourceResource, + PolicyName: "generate-policy", }, } diff --git a/test/e2e/generate/generate_test.go b/test/e2e/generate/generate_test.go index 687d94b2b8..c476758964 100644 --- a/test/e2e/generate/generate_test.go +++ b/test/e2e/generate/generate_test.go @@ -1,7 +1,6 @@ package generate import ( - "bytes" "errors" "fmt" "os" @@ -90,9 +89,25 @@ func Test_ClusterRole_ClusterRoleBinding_Sets(t *testing.T) { // ======== Create ClusterRole Policy ============= By(fmt.Sprintf("Creating Generate Role Policy in %s", clPolNS)) + loc, _ := time.LoadLocation("UTC") + timeBeforePolicyCreation := time.Now().In(loc) _, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, clPolNS, tests.Data) Expect(err).NotTo(HaveOccurred()) // ============================================ + // check policy in metrics + policySyncBool := false + e2e.GetWithRetry(time.Duration(2), 10, func() error { + metricsString, err := commonE2E.CallMetrics() + if err != nil { + return err + } + policySyncBool = commonE2E.ProcessMetrics(metricsString, tests.PolicyName, timeBeforePolicyCreation) + if policySyncBool == false { + return errors.New("policy not created") + } + return nil + }) + Expect(policySyncBool).To(Equal(true)) // == If Clone is true Create Source Resources ====== if tests.Clone { @@ -211,8 +226,26 @@ func Test_Role_RoleBinding_Sets(t *testing.T) { // ======== Create Role Policy ============= By(fmt.Sprintf("\nCreating Generate Role Policy in %s", clPolNS)) + loc, _ := time.LoadLocation("UTC") + timeBeforePolicyCreation := time.Now().In(loc) _, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, clPolNS, tests.Data) Expect(err).NotTo(HaveOccurred()) + + // check policy in metrics + policySyncBool := false + e2e.GetWithRetry(time.Duration(2), 10, func() error { + metricsString, err := commonE2E.CallMetrics() + if err != nil { + return err + } + policySyncBool = commonE2E.ProcessMetrics(metricsString, tests.PolicyName, timeBeforePolicyCreation) + if policySyncBool == false { + return errors.New("policy not created") + } + return nil + }) + Expect(policySyncBool).To(Equal(true)) + // ============================================ // === If Clone is true Create Source Resources == @@ -298,9 +331,9 @@ func Test_Role_RoleBinding_Sets(t *testing.T) { func Test_Generate_NetworkPolicy(t *testing.T) { RegisterTestingT(t) - // if os.Getenv("E2E") == "" { - // t.Skip("Skipping E2E Test") - // } + if os.Getenv("E2E") == "" { + t.Skip("Skipping E2E Test") + } // Generate E2E Client ================== e2eClient, err := e2e.NewE2EClient() Expect(err).To(BeNil()) @@ -328,19 +361,22 @@ func Test_Generate_NetworkPolicy(t *testing.T) { }) // ==================================== + // ======== Create Generate NetworkPolicy Policy ============= By("Creating Generate NetworkPolicy Policy") - // timeBeforePolicyCreation := time.Now() loc, _ := time.LoadLocation("UTC") timeBeforePolicyCreation := time.Now().In(loc) _, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, npPolNS, test.Data) Expect(err).NotTo(HaveOccurred()) // ============================================ - // check metrics + // check policy in metrics policySyncBool := false e2e.GetWithRetry(time.Duration(2), 10, func() error { - metricsString := callMetrics() + metricsString, err := commonE2E.CallMetrics() + if err != nil { + return err + } policySyncBool = commonE2E.ProcessMetrics(metricsString, test.PolicyName, timeBeforePolicyCreation) if policySyncBool == false { return errors.New("policy not created") @@ -398,19 +434,6 @@ func Test_Generate_NetworkPolicy(t *testing.T) { } } -func callMetrics() string { - requestObj := e2e.APIRequest{ - URL: "http://localhost:8000/metrics", - Type: "GET", - } - response, err := e2e.CallAPI(requestObj) - Expect(err).NotTo(HaveOccurred()) - buf := new(bytes.Buffer) - buf.ReadFrom(response.Body) - newStr := buf.String() - return newStr -} - func Test_Generate_Namespace_Label_Actions(t *testing.T) { RegisterTestingT(t) if os.Getenv("E2E") == "" { @@ -446,10 +469,27 @@ func Test_Generate_Namespace_Label_Actions(t *testing.T) { // ======== Create Generate NetworkPolicy Policy ============= By("Creating Generate NetworkPolicy Policy") + loc, _ := time.LoadLocation("UTC") + timeBeforePolicyCreation := time.Now().In(loc) _, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, npPolNS, test.Data) Expect(err).NotTo(HaveOccurred()) // ============================================ + // check policy in metrics + policySyncBool := false + e2e.GetWithRetry(time.Duration(2), 10, func() error { + metricsString, err := commonE2E.CallMetrics() + if err != nil { + return err + } + policySyncBool = commonE2E.ProcessMetrics(metricsString, test.GeneratePolicyName, timeBeforePolicyCreation) + if policySyncBool == false { + return errors.New("policy not created") + } + return nil + }) + Expect(policySyncBool).To(Equal(true)) + // Test: when creating the new namespace without the label, there should not have any generated resource // ======= Create Namespace ================== By(fmt.Sprintf("Creating Namespace which should not triggers generate policy %s", npPolNS)) @@ -633,10 +673,27 @@ func Test_Generate_Synchronize_Flag(t *testing.T) { // ==================================== // ======== Create Generate NetworkPolicy Policy ============= By("Creating Generate NetworkPolicy Policy") + loc, _ := time.LoadLocation("UTC") + timeBeforePolicyCreation := time.Now().In(loc) _, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, npPolNS, test.Data) Expect(err).NotTo(HaveOccurred()) // ================================================ + // check policy in metrics + policySyncBool := false + e2e.GetWithRetry(time.Duration(2), 10, func() error { + metricsString, err := commonE2E.CallMetrics() + if err != nil { + return err + } + policySyncBool = commonE2E.ProcessMetrics(metricsString, test.GeneratePolicyName, timeBeforePolicyCreation) + if policySyncBool == false { + return errors.New("policy not created") + } + return nil + }) + Expect(policySyncBool).To(Equal(true)) + // ======= Create Namespace ================== By(fmt.Sprintf("Creating Namespace which triggers generate %s", npPolNS)) _, err = e2eClient.CreateClusteredResourceYaml(nsGVR, namespaceWithLabelYaml) @@ -819,10 +876,27 @@ func Test_Source_Resource_Update_Replication(t *testing.T) { // ======== Create Generate Policy ============= By(fmt.Sprintf("\nCreating Generate Policy in %s", clPolNS)) + loc, _ := time.LoadLocation("UTC") + timeBeforePolicyCreation := time.Now().In(loc) _, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, clPolNS, tests.Data) Expect(err).NotTo(HaveOccurred()) // ============================================ + // check policy in metrics + policySyncBool := false + e2e.GetWithRetry(time.Duration(2), 10, func() error { + metricsString, err := commonE2E.CallMetrics() + if err != nil { + return err + } + policySyncBool = commonE2E.ProcessMetrics(metricsString, tests.PolicyName, timeBeforePolicyCreation) + if policySyncBool == false { + return errors.New("policy not created") + } + return nil + }) + Expect(policySyncBool).To(Equal(true)) + // ======= Create Namespace ================== By(fmt.Sprintf("Creating Namespace which triggers generate %s", clPolNS)) _, err = e2eClient.CreateClusteredResourceYaml(nsGVR, namespaceYaml) diff --git a/test/e2e/metrics/metrics_test.go b/test/e2e/metrics/metrics_test.go index db519b5462..c6fa7355e2 100644 --- a/test/e2e/metrics/metrics_test.go +++ b/test/e2e/metrics/metrics_test.go @@ -1,12 +1,8 @@ package metrics import ( - "bytes" - "fmt" "os" - "strings" "testing" - "time" "github.com/kyverno/kyverno/test/e2e" . "github.com/onsi/gomega" @@ -23,64 +19,5 @@ func Test_MetricsServerAvailability(t *testing.T) { } response, err := e2e.CallAPI(requestObj) Expect(err).NotTo(HaveOccurred()) - buf := new(bytes.Buffer) - buf.ReadFrom(response.Body) - newStr := buf.String() - - layout := "2006-01-02 15:04:05 -0700 MST" - timeInTimeFormat, err := time.Parse(layout, "2021-06-20 18:04:50 +0000 UTC") - if err != nil { - fmt.Println("error occurred: ", err) - } - processMetrics(newStr, "multi-tenancy", timeInTimeFormat) Expect(response.StatusCode).To(Equal(200)) } - -func processMetrics(newStr, e2ePolicyName string, e2eTime time.Time) { - fmt.Println("e2eTime: ", e2eTime) - var action, policyName string - var timeInTimeFormat time.Time - var err error - splitByNewLine := strings.Split(newStr, "\n") - for _, lineSplitedByNewLine := range splitByNewLine { - if strings.HasPrefix(lineSplitedByNewLine, "kyverno_policy_changes_info{") { - // fmt.Println(lineSplitedByNewLine) - splitByComma := strings.Split(lineSplitedByNewLine, ",") - for _, lineSplitedByComma := range splitByComma { - // fmt.Println(lineSplitedByComma) - if strings.HasPrefix(lineSplitedByComma, "policy_change_type=") { - // action = lineSplitedByComma - splitByQuote := strings.Split(lineSplitedByComma, "\"") - action = splitByQuote[1] - } - if strings.HasPrefix(lineSplitedByComma, "policy_name=") { - splitByQuote := strings.Split(lineSplitedByComma, "\"") - policyName = splitByQuote[1] - } - if strings.HasPrefix(lineSplitedByComma, "timestamp=") { - splitByQuote := strings.Split(lineSplitedByComma, "\"") - layout := "2006-01-02 15:04:05 -0700 MST" - timeInTimeFormat, err = time.Parse(layout, splitByQuote[1]) - if err != nil { - fmt.Println("error occurred: ", err) - } - } - } - if policyName == e2ePolicyName { - diff := e2eTime.Sub(timeInTimeFormat) - // fmt.Println(diff) - if diff < 0 { - // fmt.Println("-------less------") - if action == "created" { - fmt.Println("************policy created**************") - break - } - } - } - } - } - fmt.Println("action: ", action) - fmt.Println("policyName: ", policyName) - fmt.Println("timeInTimeFormat: ", timeInTimeFormat) - -} diff --git a/test/e2e/mutate/mutate_test.go b/test/e2e/mutate/mutate_test.go index c8af4be260..042274c3dd 100644 --- a/test/e2e/mutate/mutate_test.go +++ b/test/e2e/mutate/mutate_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/kyverno/kyverno/test/e2e" + commonE2E "github.com/kyverno/kyverno/test/e2e/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -79,9 +80,26 @@ func Test_Mutate_Sets(t *testing.T) { // Create CM Policy By(fmt.Sprintf("\nCreating Mutate ConfigMap Policy in %s", clPolNS)) + loc, _ := time.LoadLocation("UTC") + timeBeforePolicyCreation := time.Now().In(loc) _, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, clPolNS, tests.Data) Expect(err).NotTo(HaveOccurred()) + // check policy in metrics + policySyncBool := false + e2e.GetWithRetry(time.Duration(2), 10, func() error { + metricsString, err := commonE2E.CallMetrics() + if err != nil { + return err + } + policySyncBool = commonE2E.ProcessMetrics(metricsString, tests.PolicyName, timeBeforePolicyCreation) + if policySyncBool == false { + return errors.New("policy not created") + } + return nil + }) + Expect(policySyncBool).To(Equal(true)) + // Create target CM By(fmt.Sprintf("\nCreating target ConfigMap in %s", tests.ResourceNamespace)) _, err = e2eClient.CreateNamespacedResourceYaml(cmGVR, tests.ResourceNamespace, targetConfigMapYaml) @@ -152,9 +170,26 @@ func Test_Mutate_Ingress(t *testing.T) { Expect(err).To(BeNil()) By(fmt.Sprintf("Creating mutate ClusterPolicy ")) + loc, _ := time.LoadLocation("UTC") + timeBeforePolicyCreation := time.Now().In(loc) _, err = e2eClient.CreateClusteredResourceYaml(clPolGVR, ingressTests.cpol) Expect(err).NotTo(HaveOccurred()) + // check policy in metrics + policySyncBool := false + e2e.GetWithRetry(time.Duration(2), 10, func() error { + metricsString, err := commonE2E.CallMetrics() + if err != nil { + return err + } + policySyncBool = commonE2E.ProcessMetrics(metricsString, ingressTests.policyName, timeBeforePolicyCreation) + if policySyncBool == false { + return errors.New("policy not created") + } + return nil + }) + Expect(policySyncBool).To(Equal(true)) + By(fmt.Sprintf("Creating Namespace %s", nspace)) _, err = e2eClient.CreateClusteredResourceYaml(nsGVR, newNamespaceYaml(nspace)) Expect(err).NotTo(HaveOccurred())