Merge pull request #1001 from marquiz/devel/e2e-nodefeature

test/e2e: add basic e2e-tests for NodeFeature API

test/e2e/node_feature_discovery.go

@@ -153,14 +153,32 @@ func cleanupCRDs(cli *nfdclient.Clientset) {
 var _ = SIGDescribe("Node Feature Discovery", func() {
 	f := framework.NewDefaultFramework("node-feature-discovery")
 
+	nfdTestSuite := func(useNodeFeatureApi bool) {
+		createPodSpecOpts := func(opts ...testpod.SpecOption) []testpod.SpecOption {
+			if useNodeFeatureApi {
+				return append(opts, testpod.SpecWithContainerExtraArgs("-enable-nodefeature-api"))
+			}
+			return opts
+		}
+
 		Context("when deploying a single nfd-master pod", Ordered, func() {
 			var (
-			masterPod *corev1.Pod
 				crds      []*apiextensionsv1.CustomResourceDefinition
 				extClient *extclient.Clientset
 				nfdClient *nfdclient.Clientset
 			)
 
+			checkNodeFeatureObject := func(name string) {
+				_, err := nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Get(context.TODO(), name, metav1.GetOptions{})
+				if useNodeFeatureApi {
+					By(fmt.Sprintf("Check that NodeFeature object for the node %q was created", name))
+					Expect(err).NotTo(HaveOccurred())
+				} else {
+					By(fmt.Sprintf("Check that NodeFeature object for the node %q hasn't been created", name))
+					Expect(err).To(HaveOccurred())
+				}
+			}
+
 			BeforeAll(func() {
 				// Create clients for apiextensions and our CRD api
 				extClient = extclient.NewForConfigOrDie(f.ClientConfig())
@@ -197,13 +215,12 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
 
 				// Launch nfd-master
 				By("Creating nfd master pod and nfd-master service")
-
+				podSpecOpts := createPodSpecOpts(
-			imageOpt := []testpod.SpecOption{
 					testpod.SpecWithContainerImage(dockerImage),
 					testpod.SpecWithTolerations(testTolerations),
 					testpod.SpecWithContainerExtraArgs("-enable-taints"),
-			}
+				)
-			masterPod = e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(imageOpt...))
+				masterPod := e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(podSpecOpts...))
 
 				// Create nfd-master service
 				nfdSvc, err := testutils.CreateService(f.ClientSet, f.Namespace.Name)
@@ -246,12 +263,12 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
 
 					// Launch nfd-worker
 					By("Creating a nfd worker pod")
-				podSpecOpts := []testpod.SpecOption{
+					podSpecOpts := createPodSpecOpts(
 						testpod.SpecWithRestartPolicy(corev1.RestartPolicyNever),
 						testpod.SpecWithContainerImage(dockerImage),
 						testpod.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"),
 						testpod.SpecWithTolerations(testTolerations),
-				}
+					)
 					workerPod := testpod.NFDWorker(podSpecOpts...)
 					workerPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), workerPod, metav1.CreateOptions{})
 					Expect(err).NotTo(HaveOccurred())
@@ -275,6 +292,8 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
 						}
 					}
 
+					checkNodeFeatureObject(node.Name)
+
 					By("Deleting the node-feature-discovery worker pod")
 					err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), workerPod.ObjectMeta.Name, metav1.DeleteOptions{})
 					Expect(err).NotTo(HaveOccurred())
@@ -298,10 +317,10 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
 					fConf := cfg.DefaultFeatures
 
 					By("Creating nfd-worker daemonset")
-				podSpecOpts := []testpod.SpecOption{
+					podSpecOpts := createPodSpecOpts(
 						testpod.SpecWithContainerImage(dockerImage),
 						testpod.SpecWithTolerations(testTolerations),
-				}
+					)
 					workerDS := testds.NFDWorker(podSpecOpts...)
 					workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{})
 					Expect(err).NotTo(HaveOccurred())
@@ -363,6 +382,9 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
 							}
 						}
 
+						// Check existence of NodeFeature object
+						checkNodeFeatureObject(node.Name)
+
 					}
 
 					By("Deleting nfd-worker daemonset")
@@ -425,12 +447,12 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
 					Expect(err).NotTo(HaveOccurred())
 
 					By("Creating nfd-worker daemonset with configmap mounted")
-				podSpecOpts := []testpod.SpecOption{
+					podSpecOpts := createPodSpecOpts(
 						testpod.SpecWithContainerImage(dockerImage),
 						testpod.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")),
 						testpod.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")),
 						testpod.SpecWithTolerations(testTolerations),
-				}
+					)
 					workerDS := testds.NFDWorker(podSpecOpts...)
 
 					workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{})
@@ -488,11 +510,11 @@ core:
 					Expect(err).NotTo(HaveOccurred())
 
 					By("Creating nfd-worker daemonset")
-				podSpecOpts := []testpod.SpecOption{
+					podSpecOpts := createPodSpecOpts(
 						testpod.SpecWithContainerImage(dockerImage),
 						testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"),
 						testpod.SpecWithTolerations(testTolerations),
-				}
+					)
 					workerDS := testds.NFDWorker(podSpecOpts...)
 					workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{})
 					Expect(err).NotTo(HaveOccurred())
@@ -572,6 +594,18 @@ core:
 				})
 			})
 		})
+	}
+
+	// Run the actual tests
+	Context("when running NFD with gRPC API enabled", func() {
+		nfdTestSuite(false)
+
+	})
+
+	Context("when running NFD with NodeFeature CRD API enabled", func() {
+		nfdTestSuite(true)
+	})
+
 })
 
 // waitForNfdNodeAnnotations waits for node to be annotated as expected.

test/e2e/utils/pod/pod.go

@@ -307,7 +307,7 @@ func nfdWorkerSpec(opts ...SpecOption) *corev1.PodSpec {
 				},
 			},
 		},
-		ServiceAccountName: "nfd-master-e2e",
+		ServiceAccountName: "nfd-worker-e2e",
 		DNSPolicy:          corev1.DNSClusterFirstWithHostNet,
 		Volumes: []corev1.Volume{
 			{

test/e2e/utils/rbac.go

@@ -32,12 +32,17 @@ var (
 
 // ConfigureRBAC creates required RBAC configuration
 func ConfigureRBAC(cs clientset.Interface, ns string) error {
-	_, err := createServiceAccountMaster(cs, ns)
+	_, err := createServiceAccount(cs, "nfd-master-e2e", ns)
 	if err != nil {
 		return err
 	}
 
-	_, err = createServiceAccountTopologyUpdater(cs, ns)
+	_, err = createServiceAccount(cs, "nfd-worker-e2e", ns)
+	if err != nil {
+		return err
+	}
+
+	_, err = createServiceAccount(cs, "nfd-topology-updater-e2e", ns)
 	if err != nil {
 		return err
 	}
@@ -47,6 +52,11 @@ func ConfigureRBAC(cs clientset.Interface, ns string) error {
 		return err
 	}
 
+	_, err = createRoleWorker(cs, ns)
+	if err != nil {
+		return err
+	}
+
 	_, err = createClusterRoleTopologyUpdater(cs)
 	if err != nil {
 		return err
@@ -57,6 +67,11 @@ func ConfigureRBAC(cs clientset.Interface, ns string) error {
 		return err
 	}
 
+	_, err = createRoleBindingWorker(cs, ns)
+	if err != nil {
+		return err
+	}
+
 	_, err = createClusterRoleBindingTopologyUpdater(cs, ns)
 	if err != nil {
 		return err
@@ -75,6 +90,10 @@ func DeconfigureRBAC(cs clientset.Interface, ns string) error {
 	if err != nil {
 		return err
 	}
+	err = cs.RbacV1().RoleBindings(ns).Delete(context.TODO(), "nfd-worker-e2e", metav1.DeleteOptions{})
+	if err != nil {
+		return err
+	}
 	err = cs.RbacV1().ClusterRoles().Delete(context.TODO(), "nfd-topology-updater-e2e", metav1.DeleteOptions{})
 	if err != nil {
 		return err
@@ -83,6 +102,10 @@ func DeconfigureRBAC(cs clientset.Interface, ns string) error {
 	if err != nil {
 		return err
 	}
+	err = cs.RbacV1().Roles(ns).Delete(context.TODO(), "nfd-worker-e2e", metav1.DeleteOptions{})
+	if err != nil {
+		return err
+	}
 	err = cs.CoreV1().ServiceAccounts(ns).Delete(context.TODO(), "nfd-topology-updater-e2e", metav1.DeleteOptions{})
 	if err != nil {
 		return err
@@ -91,25 +114,18 @@ func DeconfigureRBAC(cs clientset.Interface, ns string) error {
 	if err != nil {
 		return err
 	}
+	err = cs.CoreV1().ServiceAccounts(ns).Delete(context.TODO(), "nfd-worker-e2e", metav1.DeleteOptions{})
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
-// Configure service account required by NFD Master
+// Configure service account
-func createServiceAccountMaster(cs clientset.Interface, ns string) (*corev1.ServiceAccount, error) {
+func createServiceAccount(cs clientset.Interface, name, ns string) (*corev1.ServiceAccount, error) {
 	sa := &corev1.ServiceAccount{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "nfd-master-e2e",
+			Name:      name,
-			Namespace: ns,
-		},
-	}
-	return cs.CoreV1().ServiceAccounts(ns).Create(context.TODO(), sa, metav1.CreateOptions{})
-}
-
-// Configure service account required by NFD MTopology Updater
-func createServiceAccountTopologyUpdater(cs clientset.Interface, ns string) (*corev1.ServiceAccount, error) {
-	sa := &corev1.ServiceAccount{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "nfd-topology-updater-e2e",
 			Namespace: ns,
 		},
 	}
@@ -126,11 +142,11 @@ func createClusterRoleMaster(cs clientset.Interface) (*rbacv1.ClusterRole, error
 			{
 				APIGroups: []string{""},
 				Resources: []string{"nodes"},
-				Verbs:     []string{"get", "patch", "update"},
+				Verbs:     []string{"get", "list", "patch", "update"},
 			},
 			{
 				APIGroups: []string{"nfd.k8s-sigs.io"},
-				Resources: []string{"nodefeaturerules"},
+				Resources: []string{"nodefeatures", "nodefeaturerules"},
 				Verbs:     []string{"get", "list", "watch"},
 			},
 		},
@@ -148,6 +164,24 @@ func createClusterRoleMaster(cs clientset.Interface) (*rbacv1.ClusterRole, error
 	return cs.RbacV1().ClusterRoles().Update(context.TODO(), cr, metav1.UpdateOptions{})
 }
 
+// Configure role required by NFD Worker
+func createRoleWorker(cs clientset.Interface, ns string) (*rbacv1.Role, error) {
+	cr := &rbacv1.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "nfd-worker-e2e",
+			Namespace: ns,
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups: []string{"nfd.k8s-sigs.io"},
+				Resources: []string{"nodefeatures"},
+				Verbs:     []string{"create", "get", "update"},
+			},
+		},
+	}
+	return cs.RbacV1().Roles(ns).Update(context.TODO(), cr, metav1.UpdateOptions{})
+}
+
 // Configure cluster role required by NFD Topology Updater
 func createClusterRoleTopologyUpdater(cs clientset.Interface) (*rbacv1.ClusterRole, error) {
 	cr := &rbacv1.ClusterRole{
@@ -210,6 +244,30 @@ func createClusterRoleBindingMaster(cs clientset.Interface, ns string) (*rbacv1.
 	return cs.RbacV1().ClusterRoleBindings().Update(context.TODO(), crb, metav1.UpdateOptions{})
 }
 
+// Configure role binding required by NFD Master
+func createRoleBindingWorker(cs clientset.Interface, ns string) (*rbacv1.RoleBinding, error) {
+	crb := &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "nfd-worker-e2e",
+			Namespace: ns,
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      rbacv1.ServiceAccountKind,
+				Name:      "nfd-worker-e2e",
+				Namespace: ns,
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: rbacv1.GroupName,
+			Kind:     "Role",
+			Name:     "nfd-worker-e2e",
+		},
+	}
+
+	return cs.RbacV1().RoleBindings(ns).Update(context.TODO(), crb, metav1.UpdateOptions{})
+}
+
 // Configure cluster role binding required by NFD Topology Updater
 func createClusterRoleBindingTopologyUpdater(cs clientset.Interface, ns string) (*rbacv1.ClusterRoleBinding, error) {
 	crb := &rbacv1.ClusterRoleBinding{