Merge pull request #2026 from NoSkillGirl/caching_endpoint

Adding endpoint check for policy creation

test/e2e/common/common.go

@@ -0,0 +1,71 @@
+package common
+
+import (
+	"bytes"
+	"strings"
+	"time"
+
+	"github.com/kyverno/kyverno/test/e2e"
+)
+
+func CallMetrics() (string, error) {
+	requestObj := e2e.APIRequest{
+		URL:  "http://localhost:8000/metrics",
+		Type: "GET",
+	}
+
+	response, err := e2e.CallAPI(requestObj)
+	if err != nil {
+		return "", err
+	}
+
+	buf := new(bytes.Buffer)
+	_, err = buf.ReadFrom(response.Body)
+	if err != nil {
+		return "", err
+	}
+
+	newStr := buf.String()
+	return newStr, nil
+}
+
+// ProcessMetrics checks the metrics log and identify if the policy is added in cache or not
+func ProcessMetrics(newStr, e2ePolicyName string, e2eTime time.Time) (bool, error) {
+	var action, policyName string
+	var timeInTimeFormat time.Time
+	var err error
+	splitByNewLine := strings.Split(newStr, "\n")
+	for _, lineSplitedByNewLine := range splitByNewLine {
+		if strings.HasPrefix(lineSplitedByNewLine, "kyverno_policy_changes_info{") {
+			splitByComma := strings.Split(lineSplitedByNewLine, ",")
+			for _, lineSplitedByComma := range splitByComma {
+				if strings.HasPrefix(lineSplitedByComma, "policy_change_type=") {
+					splitByQuote := strings.Split(lineSplitedByComma, "\"")
+					action = splitByQuote[1]
+				}
+				if strings.HasPrefix(lineSplitedByComma, "policy_name=") {
+					splitByQuote := strings.Split(lineSplitedByComma, "\"")
+					policyName = splitByQuote[1]
+				}
+				if strings.HasPrefix(lineSplitedByComma, "timestamp=") {
+					splitByQuote := strings.Split(lineSplitedByComma, "\"")
+					layout := "2006-01-02 15:04:05 -0700 MST"
+					timeInTimeFormat, err = time.Parse(layout, splitByQuote[1])
+					if err != nil {
+						return false, err
+					}
+				}
+			}
+
+			if policyName == e2ePolicyName {
+				diff := e2eTime.Sub(timeInTimeFormat)
+				if diff < time.Second {
+					if action == "created" {
+						return true, nil
+					}
+				}
+			}
+		}
+	}
+	return false, nil
+}

test/e2e/generate/config.go

@@ -23,6 +23,8 @@ var RoleTests = []struct {
 	Sync bool
 	// Data - The Yaml file of the ClusterPolicy of the ROle and RoleBinding - ([]byte{})
 	Data []byte
+	// PolicyName - Name of the Policy
+	PolicyName string
 }{
 	{
 		TestName:          "test-role-rolebinding-without-clone",
@@ -32,6 +34,7 @@ var RoleTests = []struct {
 		Clone:             false,
 		Sync:              false,
 		Data:              roleRoleBindingYamlWithSync,
+		PolicyName:        "gen-role-policy",
 	},
 	{
 		TestName:          "test-role-rolebinding-withsync-without-clone",
@@ -41,6 +44,7 @@ var RoleTests = []struct {
 		Clone:             false,
 		Sync:              true,
 		Data:              roleRoleBindingYamlWithSync,
+		PolicyName:        "gen-role-policy",
 	},
 	{
 		TestName:                   "test-role-rolebinding-with-clone",
@@ -53,6 +57,7 @@ var RoleTests = []struct {
 		CloneNamespace:             "default",
 		Sync:                       false,
 		Data:                       roleRoleBindingYamlWithClone,
+		PolicyName:                 "gen-role-policy",
 	},
 }
 
@@ -82,6 +87,8 @@ var ClusterRoleTests = []struct {
 	Sync bool
 	// Data - The Yaml file of the ClusterPolicy of the ClusterRole and ClusterRoleBinding - ([]byte{})
 	Data []byte
+	// PolicyName - Name of the Policy
+	PolicyName string
 }{
 	{
 		TestName:               "test-clusterrole-clusterrolebinding-without-clone",
@@ -91,6 +98,7 @@ var ClusterRoleTests = []struct {
 		Clone:                  false,
 		Sync:                   false,
 		Data:                   genClusterRoleYamlWithSync,
+		PolicyName:             "gen-cluster-policy",
 	},
 	{
 		TestName:               "test-clusterrole-clusterrolebinding-with-sync-without-clone",
@@ -100,6 +108,7 @@ var ClusterRoleTests = []struct {
 		Clone:                  false,
 		Sync:                   true,
 		Data:                   genClusterRoleYamlWithSync,
+		PolicyName:             "gen-cluster-policy",
 	},
 	{
 		TestName:                          "test-clusterrole-clusterrolebinding-with-sync-with-clone",
@@ -113,6 +122,7 @@ var ClusterRoleTests = []struct {
 		CloneSourceClusterRoleBindingData: baseClusterRoleBindingData,
 		Sync:                              false,
 		Data:                              genClusterRoleYamlWithSync,
+		PolicyName:                        "gen-cluster-policy",
 	},
 }
 
@@ -124,6 +134,8 @@ var NetworkPolicyGenerateTests = []struct {
 	NetworkPolicyName string
 	// ResourceNamespace - Namespace for which Resources are Created
 	ResourceNamespace string
+	// PolicyName - Name of the Policy
+	PolicyName string
 	// Clone - Set Clone Value
 	Clone bool
 	// CloneClusterRoleName
@@ -145,6 +157,7 @@ var NetworkPolicyGenerateTests = []struct {
 		TestName:          "test-generate-policy-for-namespace-with-label",
 		NetworkPolicyName: "allow-dns",
 		ResourceNamespace: "test",
+		PolicyName:        "add-networkpolicy",
 		Clone:             false,
 		Sync:              true,
 		Data:              genNetworkPolicyYaml,
@@ -251,6 +264,8 @@ var SourceResourceUpdateReplicationTests = []struct {
 	ConfigMapName string
 	// CloneSourceConfigMapData - Source ConfigMap Yaml
 	CloneSourceConfigMapData []byte
+	// PolicyName - Name of the Policy
+	PolicyName string
 }{
 	{
 		TestName:                 "test-clone-source-resource-update-replication",
@@ -261,5 +276,6 @@ var SourceResourceUpdateReplicationTests = []struct {
 		ConfigMapName:            "game-demo",
 		CloneNamespace:           "default",
 		CloneSourceConfigMapData: cloneSourceResource,
+		PolicyName:               "generate-policy",
 	},
 }

test/e2e/generate/generate_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/kyverno/kyverno/test/e2e"
+	commonE2E "github.com/kyverno/kyverno/test/e2e/common"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"sigs.k8s.io/yaml"
 
@@ -88,9 +89,25 @@ func Test_ClusterRole_ClusterRoleBinding_Sets(t *testing.T) {
 
 		// ======== Create ClusterRole Policy =============
 		By(fmt.Sprintf("Creating Generate Role Policy in %s", clPolNS))
+		loc, _ := time.LoadLocation("UTC")
+		timeBeforePolicyCreation := time.Now().In(loc)
 		_, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, clPolNS, tests.Data)
 		Expect(err).NotTo(HaveOccurred())
 		// ============================================
+		// check policy in metrics
+		policySyncBool := false
+		e2e.GetWithRetry(time.Duration(2), 10, func() error {
+			metricsString, err := commonE2E.CallMetrics()
+			if err != nil {
+				return err
+			}
+			policySyncBool, err = commonE2E.ProcessMetrics(metricsString, tests.PolicyName, timeBeforePolicyCreation)
+			if policySyncBool == false || err != nil {
+				return errors.New("policy not created")
+			}
+			return nil
+		})
+		Expect(policySyncBool).To(Equal(true))
 
 		// == If Clone is true Create Source Resources ======
 		if tests.Clone {
@@ -209,8 +226,26 @@ func Test_Role_RoleBinding_Sets(t *testing.T) {
 
 		// ======== Create Role Policy =============
 		By(fmt.Sprintf("\nCreating Generate Role Policy in %s", clPolNS))
+		loc, _ := time.LoadLocation("UTC")
+		timeBeforePolicyCreation := time.Now().In(loc)
 		_, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, clPolNS, tests.Data)
 		Expect(err).NotTo(HaveOccurred())
+
+		// check policy in metrics
+		policySyncBool := false
+		e2e.GetWithRetry(time.Duration(2), 10, func() error {
+			metricsString, err := commonE2E.CallMetrics()
+			if err != nil {
+				return err
+			}
+			policySyncBool, err = commonE2E.ProcessMetrics(metricsString, tests.PolicyName, timeBeforePolicyCreation)
+			if policySyncBool == false || err != nil {
+				return errors.New("policy not created")
+			}
+			return nil
+		})
+		Expect(policySyncBool).To(Equal(true))
+
 		// ============================================
 
 		// === If Clone is true Create Source Resources ==
@@ -324,13 +359,32 @@ func Test_Generate_NetworkPolicy(t *testing.T) {
 			}
 			return errors.New("deleting Namespace")
 		})
+
 		// ====================================
+
 		// ======== Create Generate NetworkPolicy Policy =============
 		By("Creating Generate NetworkPolicy Policy")
+		loc, _ := time.LoadLocation("UTC")
+		timeBeforePolicyCreation := time.Now().In(loc)
 		_, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, npPolNS, test.Data)
 		Expect(err).NotTo(HaveOccurred())
 		// ============================================
 
+		// check policy in metrics
+		policySyncBool := false
+		e2e.GetWithRetry(time.Duration(2), 10, func() error {
+			metricsString, err := commonE2E.CallMetrics()
+			if err != nil {
+				return err
+			}
+			policySyncBool, err = commonE2E.ProcessMetrics(metricsString, test.PolicyName, timeBeforePolicyCreation)
+			if policySyncBool == false || err != nil {
+				return errors.New("policy not created")
+			}
+			return nil
+		})
+		Expect(policySyncBool).To(Equal(true))
+
 		// ======= Create Namespace ==================
 		By(fmt.Sprintf("Creating Namespace which triggers generate %s", npPolNS))
 		_, err = e2eClient.CreateClusteredResourceYaml(nsGVR, namespaceWithLabelYaml)
@@ -415,10 +469,27 @@ func Test_Generate_Namespace_Label_Actions(t *testing.T) {
 
 		// ======== Create Generate NetworkPolicy Policy =============
 		By("Creating Generate NetworkPolicy Policy")
+		loc, _ := time.LoadLocation("UTC")
+		timeBeforePolicyCreation := time.Now().In(loc)
 		_, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, npPolNS, test.Data)
 		Expect(err).NotTo(HaveOccurred())
 		// ============================================
 
+		// check policy in metrics
+		policySyncBool := false
+		e2e.GetWithRetry(time.Duration(2), 10, func() error {
+			metricsString, err := commonE2E.CallMetrics()
+			if err != nil {
+				return err
+			}
+			policySyncBool, err = commonE2E.ProcessMetrics(metricsString, test.GeneratePolicyName, timeBeforePolicyCreation)
+			if policySyncBool == false || err != nil {
+				return errors.New("policy not created")
+			}
+			return nil
+		})
+		Expect(policySyncBool).To(Equal(true))
+
 		// Test: when creating the new namespace without the label, there should not have any generated resource
 		// ======= Create Namespace ==================
 		By(fmt.Sprintf("Creating Namespace which should not triggers generate policy %s", npPolNS))
@@ -602,10 +673,27 @@ func Test_Generate_Synchronize_Flag(t *testing.T) {
 		// ====================================
 		// ======== Create Generate NetworkPolicy Policy =============
 		By("Creating Generate NetworkPolicy Policy")
+		loc, _ := time.LoadLocation("UTC")
+		timeBeforePolicyCreation := time.Now().In(loc)
 		_, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, npPolNS, test.Data)
 		Expect(err).NotTo(HaveOccurred())
 		// ================================================
 
+		// check policy in metrics
+		policySyncBool := false
+		e2e.GetWithRetry(time.Duration(2), 10, func() error {
+			metricsString, err := commonE2E.CallMetrics()
+			if err != nil {
+				return err
+			}
+			policySyncBool, err = commonE2E.ProcessMetrics(metricsString, test.GeneratePolicyName, timeBeforePolicyCreation)
+			if policySyncBool == false || err != nil {
+				return errors.New("policy not created")
+			}
+			return nil
+		})
+		Expect(policySyncBool).To(Equal(true))
+
 		// ======= Create Namespace ==================
 		By(fmt.Sprintf("Creating Namespace which triggers generate %s", npPolNS))
 		_, err = e2eClient.CreateClusteredResourceYaml(nsGVR, namespaceWithLabelYaml)
@@ -788,10 +876,27 @@ func Test_Source_Resource_Update_Replication(t *testing.T) {
 
 		// ======== Create Generate Policy =============
 		By(fmt.Sprintf("\nCreating Generate Policy in %s", clPolNS))
+		loc, _ := time.LoadLocation("UTC")
+		timeBeforePolicyCreation := time.Now().In(loc)
 		_, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, clPolNS, tests.Data)
 		Expect(err).NotTo(HaveOccurred())
 		// ============================================
 
+		// check policy in metrics
+		policySyncBool := false
+		e2e.GetWithRetry(time.Duration(2), 10, func() error {
+			metricsString, err := commonE2E.CallMetrics()
+			if err != nil {
+				return err
+			}
+			policySyncBool, err = commonE2E.ProcessMetrics(metricsString, tests.PolicyName, timeBeforePolicyCreation)
+			if policySyncBool == false || err != nil {
+				return errors.New("policy not created")
+			}
+			return nil
+		})
+		Expect(policySyncBool).To(Equal(true))
+
 		// ======= Create Namespace ==================
 		By(fmt.Sprintf("Creating Namespace which triggers generate %s", clPolNS))
 		_, err = e2eClient.CreateClusteredResourceYaml(nsGVR, namespaceYaml)

test/e2e/metrics/metrics_test.go

@@ -1,10 +1,11 @@
 package metrics
 
 import (
-	"github.com/kyverno/kyverno/test/e2e"
-	. "github.com/onsi/gomega"
 	"os"
 	"testing"
+
+	"github.com/kyverno/kyverno/test/e2e"
+	. "github.com/onsi/gomega"
 )
 
 func Test_MetricsServerAvailability(t *testing.T) {

test/e2e/mutate/config.go

@@ -8,27 +8,33 @@ var MutateTests = []struct {
 	Data []byte
 	// ResourceNamespace - Namespace of the Resource
 	ResourceNamespace string
+	// PolicyName - Name of the Policy
+	PolicyName string
 }{
 	{
 		TestName:          "test-mutate-with-context",
 		Data:              configMapMutationYaml,
 		ResourceNamespace: "test-mutate",
+		PolicyName:        "mutate-policy",
 	},
 	{
 		TestName:          "test-mutate-with-logic-in-context",
 		Data:              configMapMutationWithContextLogicYaml,
 		ResourceNamespace: "test-mutate",
+		PolicyName:        "mutate-policy",
 	},
 	{
 		TestName:          "test-mutate-with-context-label-selection",
 		Data:              configMapMutationWithContextLabelSelectionYaml,
 		ResourceNamespace: "test-mutate",
+		PolicyName:        "mutate-policy",
 	},
 }
 
 var ingressTests = struct {
 	testNamesapce string
 	cpol          []byte
+	policyName    string
 	tests         []struct {
 		testName                          string
 		group, version, rsc, resourceName string
@@ -37,6 +43,7 @@ var ingressTests = struct {
 }{
 	testNamesapce: "test-ingress",
 	cpol:          mutateIngressCpol,
+	policyName:    "mutate-ingress-host",
 	tests: []struct {
 		testName                          string
 		group, version, rsc, resourceName string

test/e2e/mutate/mutate_test.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/kyverno/kyverno/test/e2e"
+	commonE2E "github.com/kyverno/kyverno/test/e2e/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -79,9 +80,26 @@ func Test_Mutate_Sets(t *testing.T) {
 
 		// Create CM Policy
 		By(fmt.Sprintf("\nCreating Mutate ConfigMap Policy in %s", clPolNS))
+		loc, _ := time.LoadLocation("UTC")
+		timeBeforePolicyCreation := time.Now().In(loc)
 		_, err = e2eClient.CreateNamespacedResourceYaml(clPolGVR, clPolNS, tests.Data)
 		Expect(err).NotTo(HaveOccurred())
 
+		// check policy in metrics
+		policySyncBool := false
+		e2e.GetWithRetry(time.Duration(2), 10, func() error {
+			metricsString, err := commonE2E.CallMetrics()
+			if err != nil {
+				return err
+			}
+			policySyncBool, err = commonE2E.ProcessMetrics(metricsString, tests.PolicyName, timeBeforePolicyCreation)
+			if policySyncBool == false || err != nil {
+				return errors.New("policy not created")
+			}
+			return nil
+		})
+		Expect(policySyncBool).To(Equal(true))
+
 		// Create target CM
 		By(fmt.Sprintf("\nCreating target ConfigMap in %s", tests.ResourceNamespace))
 		_, err = e2eClient.CreateNamespacedResourceYaml(cmGVR, tests.ResourceNamespace, targetConfigMapYaml)
@@ -152,9 +170,26 @@ func Test_Mutate_Ingress(t *testing.T) {
 	Expect(err).To(BeNil())
 
 	By(fmt.Sprintf("Creating mutate ClusterPolicy "))
+	loc, _ := time.LoadLocation("UTC")
+	timeBeforePolicyCreation := time.Now().In(loc)
 	_, err = e2eClient.CreateClusteredResourceYaml(clPolGVR, ingressTests.cpol)
 	Expect(err).NotTo(HaveOccurred())
 
+	// check policy in metrics
+	policySyncBool := false
+	e2e.GetWithRetry(time.Duration(2), 10, func() error {
+		metricsString, err := commonE2E.CallMetrics()
+		if err != nil {
+			return err
+		}
+		policySyncBool, err = commonE2E.ProcessMetrics(metricsString, ingressTests.policyName, timeBeforePolicyCreation)
+		if policySyncBool == false || err != nil {
+			return errors.New("policy not created")
+		}
+		return nil
+	})
+	Expect(policySyncBool).To(Equal(true))
+
 	By(fmt.Sprintf("Creating Namespace %s", nspace))
 	_, err = e2eClient.CreateClusteredResourceYaml(nsGVR, newNamespaceYaml(nspace))
 	Expect(err).NotTo(HaveOccurred())