test/e2e: add e2e test for nfd-gc

test/e2e/nfd_gc_test.go

@@ -0,0 +1,199 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	topologyv1alpha2 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha2"
+	topologyclient "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/generated/clientset/versioned"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1"
+	nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned"
+	"sigs.k8s.io/node-feature-discovery/test/e2e/utils"
+	testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils"
+	testdeploy "sigs.k8s.io/node-feature-discovery/test/e2e/utils/deployment"
+	testpod "sigs.k8s.io/node-feature-discovery/test/e2e/utils/pod"
+)
+
+// Actual test suite
+var _ = SIGDescribe("NFD GC", func() {
+	f := framework.NewDefaultFramework("nfd-gc")
+
+	Context("when deploying nfd-gc", Ordered, func() {
+		var (
+			crds           []*apiextensionsv1.CustomResourceDefinition
+			extClient      *extclient.Clientset
+			nfdClient      *nfdclient.Clientset
+			topologyClient *topologyclient.Clientset
+		)
+
+		BeforeAll(func(ctx context.Context) {
+			// Create clients for apiextensions and our CRD api
+			extClient = extclient.NewForConfigOrDie(f.ClientConfig())
+			nfdClient = nfdclient.NewForConfigOrDie(f.ClientConfig())
+			topologyClient = topologyclient.NewForConfigOrDie(f.ClientConfig())
+
+			By("Creating CRDs")
+			var err error
+			crds, err = testutils.CreateNfdCRDs(ctx, extClient)
+			Expect(err).NotTo(HaveOccurred())
+			crd, err := testutils.CreateNodeResourceTopologies(ctx, extClient)
+			Expect(err).NotTo(HaveOccurred())
+			crds = append(crds, crd)
+		})
+
+		AfterAll(func(ctx context.Context) {
+			for _, crd := range crds {
+				err := extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, crd.Name, metav1.DeleteOptions{})
+				Expect(err).NotTo(HaveOccurred())
+			}
+		})
+
+		JustBeforeEach(func(ctx context.Context) {
+			err := testutils.ConfigureRBAC(ctx, f.ClientSet, f.Namespace.Name)
+			Expect(err).NotTo(HaveOccurred())
+			cleanupCRs(ctx, nfdClient, f.Namespace.Name)
+			cleanupNRTs(ctx, topologyClient)
+		})
+
+		AfterEach(func(ctx context.Context) {
+			Expect(testutils.DeconfigureRBAC(ctx, f.ClientSet, f.Namespace.Name)).NotTo(HaveOccurred())
+			cleanupCRs(ctx, nfdClient, f.Namespace.Name)
+			cleanupNRTs(ctx, topologyClient)
+		})
+
+		// Helper functions
+		createCRs := func(ctx context.Context, nodeNames []string) error {
+			for _, name := range nodeNames {
+				if err := utils.CreateNodeFeature(ctx, nfdClient, f.Namespace.Name, name, name); err != nil {
+					return err
+				}
+				if err := utils.CreateNodeResourceTopology(ctx, topologyClient, name); err != nil {
+					return err
+				}
+				framework.Logf("CREATED CRS FOR node %q", name)
+			}
+			return nil
+		}
+
+		getNodeFeatures := func(ctx context.Context) ([]v1alpha1.NodeFeature, error) {
+			nfl, err := nfdClient.NfdV1alpha1().NodeFeatures("").List(ctx, metav1.ListOptions{})
+			if err != nil {
+				return nil, err
+			}
+			return nfl.Items, nil
+		}
+
+		getNodeResourceTopologies := func(ctx context.Context) ([]topologyv1alpha2.NodeResourceTopology, error) {
+			nrtl, err := topologyClient.TopologyV1alpha2().NodeResourceTopologies().List(ctx, metav1.ListOptions{})
+			if err != nil {
+				return nil, err
+			}
+			return nrtl.Items, nil
+		}
+
+		//
+		// Test GC at startup
+		//
+		Context("with pre-existing NodeFeature and NodeResourceTopology objects", func() {
+			It("it should delete stale objects at startup", func(ctx context.Context) {
+				nodes, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+				Expect(err).NotTo(HaveOccurred())
+				targetNodeNames := []string{nodes.Items[0].GetName(), nodes.Items[len(nodes.Items)-1].GetName()}
+				staleNodeNames := []string{"non-existent-node-1", "non-existent-node-2"}
+
+				// Create NodeFeature and NodeResourceTopology objects
+				By("Creating CRs")
+				Expect(createCRs(ctx, targetNodeNames)).NotTo(HaveOccurred())
+				Expect(createCRs(ctx, staleNodeNames)).NotTo(HaveOccurred())
+
+				// Deploy nfd-gc
+				By("Creating nfd-gc deployment")
+				podSpecOpts := []testpod.SpecOption{testpod.SpecWithContainerImage(dockerImage())}
+				gcDeploy := testdeploy.NFDGC(podSpecOpts...)
+				gcDeploy, err = f.ClientSet.AppsV1().Deployments(f.Namespace.Name).Create(ctx, gcDeploy, metav1.CreateOptions{})
+				Expect(err).NotTo(HaveOccurred())
+
+				By("Waiting for gc deployment pods to be ready")
+				Expect(testpod.WaitForReady(ctx, f.ClientSet, f.Namespace.Name, gcDeploy.Spec.Template.Labels["name"], 2)).NotTo(HaveOccurred())
+
+				// Check that only expected objects exist
+				By("Verifying CRs")
+				Eventually(getNodeFeatures).WithPolling(1 * time.Second).WithTimeout(3 * time.Second).WithContext(ctx).Should(ConsistOf(haveNames(targetNodeNames...)...))
+				Eventually(getNodeResourceTopologies).WithPolling(1 * time.Second).WithTimeout(3 * time.Second).WithContext(ctx).Should(ConsistOf(haveNames(targetNodeNames...)...))
+			})
+		})
+
+		//
+		// Test periodic GC
+		//
+		Context("with stale NodeFeature and NodeResourceTopology objects appearing", func() {
+			It("it should remove stale objects", func(ctx context.Context) {
+				nodes, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+				Expect(err).NotTo(HaveOccurred())
+				targetNodeNames := []string{nodes.Items[0].GetName(), nodes.Items[len(nodes.Items)-1].GetName()}
+				staleNodeNames := []string{"non-existent-node-2.1", "non-existent-node-2.2"}
+
+				// Deploy nfd-gc
+				By("Creating nfd-gc deployment")
+				podSpecOpts := []testpod.SpecOption{
+					testpod.SpecWithContainerImage(dockerImage()),
+					testpod.SpecWithContainerExtraArgs("-gc-interval", "1s"),
+				}
+				gcDeploy := testdeploy.NFDGC(podSpecOpts...)
+				gcDeploy, err = f.ClientSet.AppsV1().Deployments(f.Namespace.Name).Create(ctx, gcDeploy, metav1.CreateOptions{})
+				Expect(err).NotTo(HaveOccurred())
+
+				By("Waiting for gc deployment pods to be ready")
+				Expect(testpod.WaitForReady(ctx, f.ClientSet, f.Namespace.Name, gcDeploy.Spec.Template.Labels["name"], 2)).NotTo(HaveOccurred())
+
+				// Create NodeFeature and NodeResourceTopology objects
+				By("Creating CRs")
+				Expect(createCRs(ctx, targetNodeNames)).NotTo(HaveOccurred())
+				Expect(createCRs(ctx, staleNodeNames)).NotTo(HaveOccurred())
+
+				// Check that only expected objects exist
+				By("Verifying CRs")
+				Eventually(getNodeFeatures).WithPolling(1 * time.Second).WithTimeout(3 * time.Second).WithContext(ctx).Should(ConsistOf(haveNames(targetNodeNames...)...))
+				Eventually(getNodeResourceTopologies).WithPolling(1 * time.Second).WithTimeout(3 * time.Second).WithContext(ctx).Should(ConsistOf(haveNames(targetNodeNames...)...))
+			})
+		})
+	})
+})
+
+// haveNames is a helper that returns a slice of Gomega matchers for asserting the names of k8s API objects
+func haveNames(names ...string) []interface{} {
+	m := make([]interface{}, len(names))
+	for i, n := range names {
+		m[i] = HaveField("Name", n)
+	}
+	return m
+}
+
+func cleanupNRTs(ctx context.Context, cli *topologyclient.Clientset) {
+	By("Deleting NodeResourceTopology objects from the cluster")
+	err := cli.TopologyV1alpha2().NodeResourceTopologies().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{})
+	Expect(err).NotTo(HaveOccurred())
+}

test/e2e/utils/crd.go

@@ -144,6 +144,18 @@ func UpdateNodeFeatureRulesFromFile(ctx context.Context, cli nfdclientset.Interf
 	return nil
 }
 
+// CreateNodeFeature creates a dummy NodeFeature object for a node
+func CreateNodeFeature(ctx context.Context, cli nfdclientset.Interface, namespace, name, nodeName string) error {
+	nr := &nfdv1alpha1.NodeFeature{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   name,
+			Labels: map[string]string{nfdv1alpha1.NodeFeatureObjNodeNameLabel: nodeName},
+		},
+	}
+	_, err := cli.NfdV1alpha1().NodeFeatures(namespace).Create(ctx, nr, metav1.CreateOptions{})
+	return err
+}
+
 func apiObjsFromFile(path string, decoder apiruntime.Decoder) ([]apiruntime.Object, error) {
 	data, err := os.ReadFile(path)
 	if err != nil {

test/e2e/utils/deployment/deployment.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package daemonset
+
+import (
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"sigs.k8s.io/node-feature-discovery/test/e2e/utils/pod"
+)
+
+// NFDGC returns a deplooyment for NFD Garbage Collector
+func NFDGC(opts ...pod.SpecOption) *appsv1.Deployment {
+	return new("nfd-gc", pod.NFDGCSpec(opts...))
+}
+
+func new(name string, podSpec *corev1.PodSpec) *appsv1.Deployment {
+	return &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"name": name},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"name": name},
+				},
+				Spec: *podSpec,
+			},
+			MinReadySeconds: 1,
+		},
+	}
+}

test/e2e/utils/noderesourcetopology.go

@@ -107,6 +107,16 @@ func CreateNodeResourceTopologies(ctx context.Context, extClient extclient.Inter
 	return extClient.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, crd, metav1.CreateOptions{})
 }
 
+// CreateNodeResourceTopology creates a dummy NodeResourceTopology object for a node
+func CreateNodeResourceTopology(ctx context.Context, topologyClient *topologyclientset.Clientset, nodeName string) error {
+	nrt := &v1alpha2.NodeResourceTopology{
+		ObjectMeta: metav1.ObjectMeta{Name: nodeName},
+		Zones:      v1alpha2.ZoneList{},
+	}
+	_, err := topologyClient.TopologyV1alpha2().NodeResourceTopologies().Create(ctx, nrt, metav1.CreateOptions{})
+	return err
+}
+
 // GetNodeTopology returns the NodeResourceTopology data for the node identified by `nodeName`.
 func GetNodeTopology(ctx context.Context, topologyClient *topologyclientset.Clientset, nodeName string) *v1alpha2.NodeResourceTopology {
 	var nodeTopology *v1alpha2.NodeResourceTopology

test/e2e/utils/pod/pod.go

@@ -355,6 +355,38 @@ func nfdWorkerSpec(opts ...SpecOption) *corev1.PodSpec {
 	return p
 }
 
+func NFDGCSpec(opts ...SpecOption) *corev1.PodSpec {
+	yes := true
+	no := false
+	p := &corev1.PodSpec{
+		Containers: []corev1.Container{
+			{
+				Name:            "node-feature-discovery",
+				ImagePullPolicy: pullPolicy(),
+				Command:         []string{"nfd-gc"},
+				SecurityContext: &corev1.SecurityContext{
+					Capabilities: &corev1.Capabilities{
+						Drop: []corev1.Capability{"ALL"},
+					},
+					Privileged:               &no,
+					RunAsNonRoot:             &yes,
+					ReadOnlyRootFilesystem:   &yes,
+					AllowPrivilegeEscalation: &no,
+					SeccompProfile: &corev1.SeccompProfile{
+						Type: corev1.SeccompProfileTypeRuntimeDefault,
+					},
+				},
+			},
+		},
+		ServiceAccountName: "nfd-gc-e2e",
+	}
+
+	for _, o := range opts {
+		o(p)
+	}
+	return p
+}
+
 func NFDTopologyUpdaterSpec(kc utils.KubeletConfig, opts ...SpecOption) *corev1.PodSpec {
 	p := &corev1.PodSpec{
 		Containers: []corev1.Container{

test/e2e/utils/rbac.go

@@ -42,6 +42,11 @@ func ConfigureRBAC(ctx context.Context, cs clientset.Interface, ns string) error
 		return err
 	}
 
+	_, err = createServiceAccount(ctx, cs, "nfd-gc-e2e", ns)
+	if err != nil {
+		return err
+	}
+
 	_, err = createServiceAccount(ctx, cs, "nfd-topology-updater-e2e", ns)
 	if err != nil {
 		return err
@@ -57,6 +62,11 @@ func ConfigureRBAC(ctx context.Context, cs clientset.Interface, ns string) error
 		return err
 	}
 
+	_, err = createClusterRoleGC(ctx, cs)
+	if err != nil {
+		return err
+	}
+
 	_, err = createClusterRoleTopologyUpdater(ctx, cs)
 	if err != nil {
 		return err
@@ -72,6 +82,11 @@ func ConfigureRBAC(ctx context.Context, cs clientset.Interface, ns string) error
 		return err
 	}
 
+	_, err = createClusterRoleBindingGC(ctx, cs, ns)
+	if err != nil {
+		return err
+	}
+
 	_, err = createClusterRoleBindingTopologyUpdater(ctx, cs, ns)
 	if err != nil {
 		return err
@@ -94,6 +109,10 @@ func DeconfigureRBAC(ctx context.Context, cs clientset.Interface, ns string) err
 	if err != nil {
 		return err
 	}
+	err = cs.RbacV1().ClusterRoleBindings().Delete(ctx, "nfd-gc-e2e", metav1.DeleteOptions{})
+	if err != nil {
+		return err
+	}
 	err = cs.RbacV1().ClusterRoles().Delete(ctx, "nfd-topology-updater-e2e", metav1.DeleteOptions{})
 	if err != nil {
 		return err
@@ -106,6 +125,10 @@ func DeconfigureRBAC(ctx context.Context, cs clientset.Interface, ns string) err
 	if err != nil {
 		return err
 	}
+	err = cs.RbacV1().ClusterRoles().Delete(ctx, "nfd-gc-e2e", metav1.DeleteOptions{})
+	if err != nil {
+		return err
+	}
 	err = cs.CoreV1().ServiceAccounts(ns).Delete(ctx, "nfd-topology-updater-e2e", metav1.DeleteOptions{})
 	if err != nil {
 		return err
@@ -118,6 +141,10 @@ func DeconfigureRBAC(ctx context.Context, cs clientset.Interface, ns string) err
 	if err != nil {
 		return err
 	}
+	err = cs.CoreV1().ServiceAccounts(ns).Delete(ctx, "nfd-gc-e2e", metav1.DeleteOptions{})
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -182,6 +209,33 @@ func createRoleWorker(ctx context.Context, cs clientset.Interface, ns string) (*
 	return cs.RbacV1().Roles(ns).Update(ctx, cr, metav1.UpdateOptions{})
 }
 
+// Configure cluster role required by NFD GC
+func createClusterRoleGC(ctx context.Context, cs clientset.Interface) (*rbacv1.ClusterRole, error) {
+	cr := &rbacv1.ClusterRole{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "nfd-gc-e2e",
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups: []string{""},
+				Resources: []string{"nodes"},
+				Verbs:     []string{"list", "watch"},
+			},
+			{
+				APIGroups: []string{"nfd.k8s-sigs.io"},
+				Resources: []string{"nodefeatures"},
+				Verbs:     []string{"list", "delete"},
+			},
+			{
+				APIGroups: []string{"topology.node.k8s.io"},
+				Resources: []string{"noderesourcetopologies"},
+				Verbs:     []string{"list", "delete"},
+			},
+		},
+	}
+	return cs.RbacV1().ClusterRoles().Update(ctx, cr, metav1.UpdateOptions{})
+}
+
 // Configure cluster role required by NFD Topology Updater
 func createClusterRoleTopologyUpdater(ctx context.Context, cs clientset.Interface) (*rbacv1.ClusterRole, error) {
 	cr := &rbacv1.ClusterRole{
@@ -268,6 +322,29 @@ func createRoleBindingWorker(ctx context.Context, cs clientset.Interface, ns str
 	return cs.RbacV1().RoleBindings(ns).Update(ctx, crb, metav1.UpdateOptions{})
 }
 
+// Configure cluster role binding required by NFD GC
+func createClusterRoleBindingGC(ctx context.Context, cs clientset.Interface, ns string) (*rbacv1.ClusterRoleBinding, error) {
+	crb := &rbacv1.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "nfd-gc-e2e",
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      rbacv1.ServiceAccountKind,
+				Name:      "nfd-gc-e2e",
+				Namespace: ns,
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: rbacv1.GroupName,
+			Kind:     "ClusterRole",
+			Name:     "nfd-gc-e2e",
+		},
+	}
+
+	return cs.RbacV1().ClusterRoleBindings().Update(ctx, crb, metav1.UpdateOptions{})
+}
+
 // Configure cluster role binding required by NFD Topology Updater
 func createClusterRoleBindingTopologyUpdater(ctx context.Context, cs clientset.Interface, ns string) (*rbacv1.ClusterRoleBinding, error) {
 	crb := &rbacv1.ClusterRoleBinding{