Run E2E tests on all supported k8s versions (#3256)

.github/workflows/e2e.yaml

@@ -21,6 +21,10 @@ permissions: read-all
 
 jobs:
   e2e-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s-version: [v1.16.15, v1.17.17, v1.18.19, v1.19.11, v1.20.7, v1.21.2, v1.22.5, v1.23.3]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -52,7 +56,8 @@ jobs:
 
       - name :  Create dev images, kind cluster and setup kustomize
         run: |
-          make -j4 create-e2e-infrastruture
+          export KIND_IMAGE=kindest/node:${{ matrix.k8s-version }}
+          make create-e2e-infrastruture
 
       - name: e2e testing
         run: |

Makefile

@@ -26,7 +26,8 @@ endif
 PACKAGE ?=github.com/kyverno/kyverno
 LD_FLAGS="-s -w -X $(PACKAGE)/pkg/version.BuildVersion=$(GIT_VERSION) -X $(PACKAGE)/pkg/version.BuildHash=$(GIT_HASH) -X $(PACKAGE)/pkg/version.BuildTime=$(TIMESTAMP)"
 LD_FLAGS_DEV="-s -w -X $(PACKAGE)/pkg/version.BuildVersion=$(GIT_VERSION_DEV) -X $(PACKAGE)/pkg/version.BuildHash=$(GIT_HASH) -X $(PACKAGE)/pkg/version.BuildTime=$(TIMESTAMP)"
-
+K8S_VERSION ?= $(shell kubectl version --short | grep -i server | cut -d" " -f3 | cut -c2-)
+export K8S_VERSION
 ##################################
 # KYVERNO
 ##################################

scripts/create-e2e-infrastruture.sh

@@ -9,7 +9,12 @@ curl -Lo $pwd/kind https://kind.sigs.k8s.io/dl/v0.11.0/kind-linux-amd64
 chmod a+x $pwd/kind
 
 ## Create Kind Cluster
-$pwd/kind create cluster
+if [ -z "${KIND_IMAGE}" ]; then
+    $pwd/kind create cluster
+else
+    $pwd/kind create cluster --image="${KIND_IMAGE}"
+fi
+
 $pwd/kind load docker-image ghcr.io/kyverno/kyverno:$hash
 $pwd/kind load docker-image ghcr.io/kyverno/kyvernopre:$hash
 

test/e2e/common/common.go

@@ -3,12 +3,16 @@ package common
 import (
 	"bytes"
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
+	"github.com/blang/semver/v4"
 	"github.com/kyverno/kyverno/test/e2e"
 )
 
+const defaultTestK8sVersion = "1.21.0"
+
 func CallMetrics() (string, error) {
 	requestObj := e2e.APIRequest{
 		URL:  "http://localhost:8000/metrics",
@@ -30,6 +34,14 @@ func CallMetrics() (string, error) {
 	return newStr, nil
 }
 
+func GetKubernetesVersion() semver.Version {
+	ver, err := semver.Parse(os.Getenv("K8S_VERSION"))
+	if err != nil {
+		return semver.MustParse(defaultTestK8sVersion)
+	}
+	return ver
+}
+
 // ProcessMetrics checks the metrics log and identify if the policy is added in cache or not
 func ProcessMetrics(newStr, e2ePolicyName string) error {
 	splitByNewLine := strings.Split(newStr, "\n")

test/e2e/mutate/config.go

@@ -1,6 +1,8 @@
 package mutate
 
 import (
+	"github.com/blang/semver/v4"
+	"github.com/kyverno/kyverno/test/e2e/common"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 
@@ -101,7 +103,7 @@ var tests = []struct {
 		PolicyName:         "replace-docker-hub",
 		PolicyRaw:          kyverno_2971_policy,
 		ResourceName:       "nginx",
-		ResourceNamespace:  "test-mutate",
+		ResourceNamespace:  "test-mutate-img",
 		ResourceGVR:        podGVR,
 		ResourceRaw:        kyverno_2971_resource,
 		ExpectedPatternRaw: kyverno_2971_pattern,
@@ -109,22 +111,24 @@ var tests = []struct {
 }
 
 var ingressTests = struct {
-	testNamesapce string
+	testNamespace string
 	cpol          []byte
 	policyName    string
 	tests         []struct {
 		testName                          string
 		group, version, rsc, resourceName string
 		resource                          []byte
+		skip                              bool
 	}
 }{
-	testNamesapce: "test-ingress",
+	testNamespace: "test-ingress",
 	cpol:          mutateIngressCpol,
 	policyName:    "mutate-ingress-host",
 	tests: []struct {
 		testName                          string
 		group, version, rsc, resourceName string
 		resource                          []byte
+		skip                              bool
 	}{
 		{
 			testName:     "test-networking-v1-ingress",
@@ -133,6 +137,7 @@ var ingressTests = struct {
 			rsc:          "ingresses",
 			resourceName: "kuard-v1",
 			resource:     ingressNetworkingV1,
+			skip:         common.GetKubernetesVersion().LT(semver.MustParse("1.19.0")),
 		},
 		// the following test can be removed after 1.22 cluster
 		{
@@ -142,6 +147,7 @@ var ingressTests = struct {
 			rsc:          "ingresses",
 			resourceName: "kuard-v1beta1",
 			resource:     ingressNetworkingV1beta1,
+			skip:         common.GetKubernetesVersion().GTE(semver.MustParse("1.22.0")),
 		},
 	},
 }

test/e2e/mutate/mutate_test.go

@@ -256,7 +256,7 @@ func Test_Mutate_Ingress(t *testing.T) {
 	e2eClient, err := e2e.NewE2EClient()
 	Expect(err).To(BeNil())
 
-	nspace := ingressTests.testNamesapce
+	nspace := ingressTests.testNamespace
 	By("Cleaning Cluster Policies")
 	e2eClient.CleanClusterPolicies(policyGVR)
 
@@ -285,8 +285,10 @@ func Test_Mutate_Ingress(t *testing.T) {
 	Expect(err).NotTo(HaveOccurred())
 
 	for _, test := range ingressTests.tests {
+		if test.skip {
+			continue
+		}
 		By(fmt.Sprintf("\n\nStart testing %s", test.testName))
-
 		gvr := e2e.GetGVR(test.group, test.version, test.rsc)
 		By(fmt.Sprintf("Creating Ingress %v in %s", gvr, nspace))
 		_, err = e2eClient.CreateNamespacedResourceYaml(gvr, nspace, test.resource)

test/e2e/mutate/resources.go

@@ -566,7 +566,7 @@ apiVersion: v1
 kind: Pod
 metadata:
   name: nginx
-  namespace: test-mutate
+  namespace: test-mutate-img
 spec:
   containers:
   - name: nginx