feat: add e2e framework and verify image new test (#4094)

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

test/e2e/framework/client/client.go

@@ -0,0 +1,92 @@
+package client
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kyverno/kyverno/test/e2e"
+	"github.com/kyverno/kyverno/test/e2e/framework/id"
+	"github.com/kyverno/kyverno/test/e2e/framework/resource"
+	"github.com/kyverno/kyverno/test/e2e/framework/utils"
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+type Client interface {
+	CreateResource(resource.Resource) *unstructured.Unstructured
+	GetResource(id.Id) *unstructured.Unstructured
+	DeleteResource(id.Id)
+}
+
+type client struct {
+	t      *testing.T
+	client *e2e.E2EClient
+}
+
+func New(t *testing.T) Client {
+	c, err := e2e.NewE2EClient()
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	return &client{t, c}
+}
+
+func (c *client) CreateResource(resource resource.Resource) *unstructured.Unstructured {
+	u := resource.Unstructured()
+	ginkgo.By(fmt.Sprintf("Creating %s : %s", resource.Gvr(), utils.Key(u)))
+	var err error
+	if u.GetNamespace() != "" {
+		u, err = c.client.CreateNamespacedResource(resource.Gvr(), u.GetNamespace(), u)
+	} else {
+		u, err = c.client.CreateClusteredResource(resource.Gvr(), u)
+	}
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	c.t.Cleanup(func() {
+		c.DeleteResource(id.New(resource.Gvr(), u.GetNamespace(), u.GetName()))
+	})
+	return u
+}
+
+func (c *client) DeleteResource(id id.Id) {
+	ginkgo.By(fmt.Sprintf("Deleting %s : %s", id.GetGvr(), utils.Key(id)))
+	var err error
+	if id.IsClustered() {
+		err = c.client.DeleteClusteredResource(id.GetGvr(), id.GetName())
+	} else {
+		err = c.client.DeleteNamespacedResource(id.GetGvr(), id.GetNamespace(), id.GetName())
+	}
+	if !apierrors.IsNotFound(err) {
+		err = e2e.GetWithRetry(1*time.Second, 15, func() error {
+			if id.IsClustered() {
+				_, err = c.client.GetClusteredResource(id.GetGvr(), id.GetName())
+			} else {
+				_, err = c.client.GetNamespacedResource(id.GetGvr(), id.GetNamespace(), id.GetName())
+			}
+			if err == nil {
+				return fmt.Errorf("resource still exists: %s, %s", id.GetGvr(), utils.Key(id))
+			}
+			if apierrors.IsNotFound(err) {
+				return nil
+			}
+			return err
+		})
+	}
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+}
+
+func (c *client) GetResource(id id.Id) *unstructured.Unstructured {
+	ginkgo.By(fmt.Sprintf("Getting %s : %s", id.GetGvr(), utils.Key(id)))
+	var u *unstructured.Unstructured
+	err := e2e.GetWithRetry(1*time.Second, 15, func() error {
+		var err error
+		if id.IsClustered() {
+			u, err = c.client.GetClusteredResource(id.GetGvr(), id.GetName())
+		} else {
+			u, err = c.client.GetNamespacedResource(id.GetGvr(), id.GetNamespace(), id.GetName())
+		}
+		return err
+	})
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	return u
+}

test/e2e/framework/framework.go

@@ -0,0 +1,27 @@
+package framework
+
+import (
+	"os"
+	"testing"
+
+	"github.com/kyverno/kyverno/test/e2e/framework/client"
+	"github.com/kyverno/kyverno/test/e2e/framework/step"
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+)
+
+func Setup(t *testing.T) {
+	gomega.RegisterTestingT(t)
+	if os.Getenv("E2E") == "" {
+		t.Skip("Skipping E2E Test")
+	}
+}
+
+func Run(t *testing.T, steps ...step.Step) {
+	ginkgo.By("Creating client ...")
+	client := client.New(t)
+	for _, step := range steps {
+		step(client)
+	}
+	ginkgo.By("Cleaning up ...")
+}

test/e2e/framework/gvr/gvr.go

@@ -0,0 +1,14 @@
+package gvr
+
+import "github.com/kyverno/kyverno/test/e2e"
+
+var (
+	ClusterPolicy      = e2e.GetGVR("kyverno.io", "v1", "clusterpolicies")
+	Namespace          = e2e.GetGVR("", "v1", "namespaces")
+	ClusterRole        = e2e.GetGVR("rbac.authorization.k8s.io", "v1", "clusterroles")
+	ClusterRoleBinding = e2e.GetGVR("rbac.authorization.k8s.io", "v1", "clusterrolebindings")
+	Role               = e2e.GetGVR("rbac.authorization.k8s.io", "v1", "roles")
+	RoleBinding        = e2e.GetGVR("rbac.authorization.k8s.io", "v1", "rolebindings")
+	ConfigMap          = e2e.GetGVR("", "v1", "configmaps")
+	NetworkPolicy      = e2e.GetGVR("networking.k8s.io", "v1", "networkpolicies")
+)

test/e2e/framework/id/clustered.go

@@ -0,0 +1,26 @@
+package id
+
+import (
+	"github.com/kyverno/kyverno/test/e2e/framework/gvr"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+func Clustered(gvr schema.GroupVersionResource, name string) Id {
+	return New(gvr, "", name)
+}
+
+func ClusterPolicy(name string) Id {
+	return Clustered(gvr.ClusterPolicy, name)
+}
+
+func ClusterRole(name string) Id {
+	return Clustered(gvr.ClusterRole, name)
+}
+
+func ClusterRoleBinding(name string) Id {
+	return Clustered(gvr.ClusterRoleBinding, name)
+}
+
+func Namespace(name string) Id {
+	return Clustered(gvr.Namespace, name)
+}

test/e2e/framework/id/id.go

@@ -0,0 +1,18 @@
+package id
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type Id struct {
+	gvr  schema.GroupVersionResource
+	ns   string
+	name string
+}
+
+func New(gvr schema.GroupVersionResource, ns, name string) Id { return Id{gvr, ns, name} }
+func (r Id) GetGvr() schema.GroupVersionResource              { return r.gvr }
+func (r Id) GetNamespace() string                             { return r.ns }
+func (r Id) GetName() string                                  { return r.name }
+func (r Id) IsClustered() bool                                { return r.ns == "" }
+func (r Id) IsNamespaced() bool                               { return !r.IsClustered() }

test/e2e/framework/resource/clustered.go

@@ -0,0 +1,26 @@
+package resource
+
+import (
+	"github.com/kyverno/kyverno/test/e2e/framework/gvr"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+func Clustered(gvr schema.GroupVersionResource, raw []byte) Resource {
+	return Resource{gvr, "", raw}
+}
+
+func ClusterPolicy(raw []byte) Resource {
+	return Clustered(gvr.ClusterPolicy, raw)
+}
+
+func ClusterRole(raw []byte) Resource {
+	return Clustered(gvr.ClusterRole, raw)
+}
+
+func ClusterRoleBinding(raw []byte) Resource {
+	return Clustered(gvr.ClusterRoleBinding, raw)
+}
+
+func Namespace(raw []byte) Resource {
+	return Clustered(gvr.Namespace, raw)
+}

test/e2e/framework/resource/namespaced.go

@@ -0,0 +1,22 @@
+package resource
+
+import (
+	"github.com/kyverno/kyverno/test/e2e/framework/gvr"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+func Namespaced(gvr schema.GroupVersionResource, ns string, raw []byte) Resource {
+	return Resource{gvr, ns, raw}
+}
+
+func Role(ns string, raw []byte) Resource {
+	return Namespaced(gvr.Role, ns, raw)
+}
+
+func RoleBinding(ns string, raw []byte) Resource {
+	return Namespaced(gvr.RoleBinding, ns, raw)
+}
+
+func ConfigMap(ns string, raw []byte) Resource {
+	return Namespaced(gvr.ConfigMap, ns, raw)
+}

test/e2e/framework/resource/resource.go

@@ -0,0 +1,32 @@
+package resource
+
+import (
+	"github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/yaml"
+)
+
+type Resource struct {
+	gvr  schema.GroupVersionResource
+	ns   string
+	data []byte
+}
+
+func (r *Resource) Gvr() schema.GroupVersionResource { return r.gvr }
+func (r *Resource) Namespace() string                { return r.ns }
+func (r *Resource) Data() []byte                     { return r.data }
+func (r *Resource) IsClustered() bool                { return r.ns == "" }
+func (r *Resource) IsNamespaced() bool               { return !r.IsClustered() }
+
+func (r *Resource) Unstructured() *unstructured.Unstructured {
+	var u unstructured.Unstructured
+	gomega.Expect(yaml.Unmarshal(r.data, &u)).To(gomega.Succeed())
+	// TODO: set namespace ?
+	// TODO: ensure GV(R/K) ?
+	return &u
+}
+
+func Resources(resources ...Resource) []Resource {
+	return resources
+}

test/e2e/framework/step/by.go

@@ -0,0 +1,12 @@
+package step
+
+import (
+	"github.com/kyverno/kyverno/test/e2e/framework/client"
+	"github.com/onsi/ginkgo"
+)
+
+func By(message string) Step {
+	return func(client.Client) {
+		ginkgo.By(message)
+	}
+}

test/e2e/framework/step/expect.go

@@ -0,0 +1,23 @@
+package step
+
+import (
+	"fmt"
+
+	"github.com/kyverno/kyverno/test/e2e/framework/client"
+	"github.com/kyverno/kyverno/test/e2e/framework/id"
+	"github.com/kyverno/kyverno/test/e2e/framework/utils"
+	"github.com/onsi/ginkgo"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+type ResourceExpectation func(*unstructured.Unstructured)
+
+func ExpectResource(id id.Id, expectations ...ResourceExpectation) Step {
+	return func(client client.Client) {
+		ginkgo.By(fmt.Sprintf("Checking resource expectations (%s : %s) ...", id.GetGvr(), utils.Key(id)))
+		resource := client.GetResource(id)
+		for _, expectation := range expectations {
+			expectation(resource)
+		}
+	}
+}

test/e2e/framework/step/policy.go

@@ -0,0 +1,17 @@
+package step
+
+import (
+	"github.com/kyverno/kyverno/test/e2e/common"
+	"github.com/kyverno/kyverno/test/e2e/framework/client"
+	"github.com/kyverno/kyverno/test/e2e/framework/resource"
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+)
+
+func CreateClusterPolicy(data []byte) Step {
+	return func(client client.Client) {
+		ginkgo.By("Creating cluster policy ...")
+		policy := client.CreateResource(resource.ClusterPolicy(data))
+		gomega.Expect(common.PolicyCreated(policy.GetName())).To(gomega.Succeed())
+	}
+}

test/e2e/framework/step/step.go

@@ -0,0 +1,7 @@
+package step
+
+import (
+	"github.com/kyverno/kyverno/test/e2e/framework/client"
+)
+
+type Step func(client.Client)

test/e2e/framework/utils/named.go

@@ -0,0 +1,14 @@
+package utils
+
+type Named interface {
+	GetNamespace() string
+	GetName() string
+}
+
+func Key(obj Named) string {
+	n, ns := obj.GetName(), obj.GetNamespace()
+	if ns == "" {
+		return n
+	}
+	return ns + "/" + n
+}

test/e2e/verifyimages/resources.go

@@ -203,3 +203,27 @@ spec:
         5/KAQN0/KjHcorm/J5yctVd7iEcnessRQjU917hmKO6JWVGHpDguIyakZA==
         -----END PUBLIC KEY----- 
 `)
+
+var cpolVerifyImages = []byte(`
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: verify-images
+spec:
+  validationFailureAction: enforce
+  rules:
+    - name: check-image-sig
+      match:
+        any:
+        - resources:
+            kinds:
+              - Pod
+      verifyImages:
+      - image: "harbor2.zoller.com/cosign/*"
+        mutateDigest: false
+        key: |-
+          -----BEGIN PUBLIC KEY-----
+          MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEpNlOGZ323zMlhs4bcKSpAKQvbcWi
+          5ZLRmijm6SqXDy0Fp0z0Eal+BekFnLzs8rUXUaXlhZ3hNudlgFJH+nFNMw==
+          -----END PUBLIC KEY-----
+`)

test/e2e/verifyimages/verifyimages_test.go

@@ -8,8 +8,12 @@ import (
 	"time"
 
 	"github.com/kyverno/kyverno/test/e2e"
+	"github.com/kyverno/kyverno/test/e2e/framework"
+	"github.com/kyverno/kyverno/test/e2e/framework/id"
+	"github.com/kyverno/kyverno/test/e2e/framework/step"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
 
 var (
@@ -149,3 +153,23 @@ func TestImageVerify(t *testing.T) {
 	e2eClient.DeleteClusteredResource(crdGVR, crdName)
 
 }
+
+func Test_BoolFields(t *testing.T) {
+	framework.Setup(t)
+	framework.Run(t,
+		step.CreateClusterPolicy(cpolVerifyImages),
+		step.By("Checking spec.rules[0].verifyImages[0].mutateDigest is false ..."),
+		step.ExpectResource(id.ClusterPolicy("verify-images"), func(resource *unstructured.Unstructured) {
+			rules, found, err := unstructured.NestedSlice(resource.UnstructuredContent(), "spec", "rules")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(found).To(BeTrue())
+			verifyImages, found, err := unstructured.NestedSlice(rules[0].(map[string]interface{}), "verifyImages")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(found).To(BeTrue())
+			mutateDigest, found, err := unstructured.NestedBool(verifyImages[0].(map[string]interface{}), "mutateDigest")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(found).To(BeTrue())
+			Expect(mutateDigest).To(BeFalse())
+		}),
+	)
+}