diff --git a/go.mod b/go.mod index 7c25f60f5..be823b2d3 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module sigs.k8s.io/node-feature-discovery go 1.22.2 +toolchain go1.22.0 + require ( github.com/fsnotify/fsnotify v1.7.0 github.com/golang/protobuf v1.5.4 diff --git a/pkg/nfd-master/nfd-api-controller.go b/pkg/nfd-master/nfd-api-controller.go index 81b5bd35e..905a2ed0c 100644 --- a/pkg/nfd-master/nfd-api-controller.go +++ b/pkg/nfd-master/nfd-api-controller.go @@ -17,6 +17,7 @@ limitations under the License. package nfdmaster import ( + "context" "fmt" "time" @@ -26,7 +27,6 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" - nfdclientset "sigs.k8s.io/node-feature-discovery/api/generated/clientset/versioned" nfdscheme "sigs.k8s.io/node-feature-discovery/api/generated/clientset/versioned/scheme" nfdinformers "sigs.k8s.io/node-feature-discovery/api/generated/informers/externalversions" nfdinformersv1alpha1 "sigs.k8s.io/node-feature-discovery/api/generated/informers/externalversions/nfd/v1alpha1" @@ -65,6 +65,8 @@ func newNfdController(config *restclient.Config, nfdApiControllerOptions nfdApiC updateOneNodeChan: make(chan string), updateAllNodeFeatureGroupsChan: make(chan struct{}, 1), updateNodeFeatureGroupChan: make(chan string), + k8sClient: nfdApiControllerOptions.K8sClient, + nodeFeatureNamespaceSelector: nfdApiControllerOptions.NodeFeatureNamespaceSelector, } nfdClient := nfdclientset.NewForConfigOrDie(config) @@ -89,25 +91,28 @@ func newNfdController(config *restclient.Config, nfdApiControllerOptions nfdApiC AddFunc: func(obj interface{}) { nfr := obj.(*nfdv1alpha1.NodeFeature) klog.V(2).InfoS("NodeFeature added", "nodefeature", klog.KObj(nfr)) - c.updateOneNode("NodeFeature", nfr) - if !nfdApiControllerOptions.DisableNodeFeatureGroup { - c.updateAllNodeFeatureGroups() + if c.isNamespaceSelected(nfr.Namespace) { + c.updateOneNode("NodeFeature", nfr) + } else { + klog.InfoS("NodeFeature not in selected namespace", "namespace", nfr.Namespace, "name", nfr.Name) } }, UpdateFunc: func(oldObj, newObj interface{}) { nfr := newObj.(*nfdv1alpha1.NodeFeature) klog.V(2).InfoS("NodeFeature updated", "nodefeature", klog.KObj(nfr)) - c.updateOneNode("NodeFeature", nfr) - if !nfdApiControllerOptions.DisableNodeFeatureGroup { - c.updateAllNodeFeatureGroups() + if c.isNamespaceSelected(nfr.Namespace) { + c.updateOneNode("NodeFeature", nfr) + } else { + klog.InfoS("NodeFeature not in selected namespace", "namespace", nfr.Namespace, "name", nfr.Name) } }, DeleteFunc: func(obj interface{}) { nfr := obj.(*nfdv1alpha1.NodeFeature) klog.V(2).InfoS("NodeFeature deleted", "nodefeature", klog.KObj(nfr)) - c.updateOneNode("NodeFeature", nfr) - if !nfdApiControllerOptions.DisableNodeFeatureGroup { - c.updateAllNodeFeatureGroups() + if c.isNamespaceSelected(nfr.Namespace) { + c.updateOneNode("NodeFeature", nfr) + } else { + klog.InfoS("NodeFeature not in selected namespace", "namespace", nfr.Namespace, "name", nfr.Name) } }, }); err != nil { @@ -209,6 +214,37 @@ func (c *nfdController) updateOneNode(typ string, obj metav1.Object) { c.updateOneNodeChan <- nodeName } +func (c *nfdController) isNamespaceSelected(namespace string) bool { + // no namespace restrictions are made here + if c.nodeFeatureNamespaceSelector == nil { + return true + } + + labelMap, err := metav1.LabelSelectorAsSelector(c.nodeFeatureNamespaceSelector) + if err != nil { + klog.ErrorS(err, "failed to convert label selector to map", "selector", c.nodeFeatureNamespaceSelector) + return false + } + + listOptions := metav1.ListOptions{ + LabelSelector: labelMap.String(), + } + + namespaces, err := c.k8sClient.CoreV1().Namespaces().List(context.Background(), listOptions) + if err != nil { + klog.ErrorS(err, "failed to query namespaces", "listOptions", listOptions) + return false + } + + for _, ns := range namespaces.Items { + if ns.Name == namespace { + return true + } + } + + return false +} + func (c *nfdController) updateAllNodes() { select { case c.updateAllNodesChan <- struct{}{}: diff --git a/pkg/nfd-master/nfd-api-controller_test.go b/pkg/nfd-master/nfd-api-controller_test.go index 44153f41a..21401d96c 100644 --- a/pkg/nfd-master/nfd-api-controller_test.go +++ b/pkg/nfd-master/nfd-api-controller_test.go @@ -23,6 +23,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/api/nfd/v1alpha1" + fakeclient "k8s.io/client-go/kubernetes/fake" + corev1 "k8s.io/api/core/v1" ) func TestGetNodeNameForObj(t *testing.T) { @@ -42,3 +44,57 @@ func TestGetNodeNameForObj(t *testing.T) { assert.Nil(t, err) assert.Equal(t, n, "node-1") } + +func newTestNamespace(name string) *corev1.Namespace{ + return &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "name": name, + }, + }, + } +} + +func TestIsNamespaceAllowed(t *testing.T) { + fakeCli := fakeclient.NewSimpleClientset(newTestNamespace("fake")) + c := &nfdController{ + k8sClient: fakeCli, + } + + testcases := []struct { + name string + objectNamespace string + nodeFeatureNamespaceSelector *metav1.LabelSelector + expectedResult bool + }{ + { + name: "namespace not allowed", + objectNamespace: "random", + nodeFeatureNamespaceSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "name", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"fake"}, + }, + }, + }, + expectedResult: false, + }, + { + name: "namespace is allowed", + objectNamespace: "fake", + nodeFeatureNamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"name": "fake"}, + }, + expectedResult: false, + }, + } + + for _, tc := range testcases { + c.nodeFeatureNamespaceSelector = tc.nodeFeatureNamespaceSelector + res := c.isNamespaceSelected(tc.name) + assert.Equal(t, res, tc.expectedResult) + } +} diff --git a/pkg/nfd-master/nfd-master-internal_test.go b/pkg/nfd-master/nfd-master-internal_test.go index 498070a86..e6e73103e 100644 --- a/pkg/nfd-master/nfd-master-internal_test.go +++ b/pkg/nfd-master/nfd-master-internal_test.go @@ -509,15 +509,16 @@ func TestFilterLabels(t *testing.T) { func TestCreatePatches(t *testing.T) { Convey("When creating JSON patches", t, func() { existingItems := map[string]string{"key-1": "val-1", "key-2": "val-2", "key-3": "val-3"} + overwriteKeys := true jsonPath := "/root" - Convey("When when there are neither itmes to remoe nor to add or update", func() { - p := createPatches([]string{"foo", "bar"}, existingItems, map[string]string{}, jsonPath) + Convey("When there are neither itmes to remoe nor to add or update", func() { + p := createPatches([]string{"foo", "bar"}, existingItems, map[string]string{}, jsonPath, overwriteKeys) So(len(p), ShouldEqual, 0) }) - Convey("When when there are itmes to remoe but none to add or update", func() { - p := createPatches([]string{"key-2", "key-3", "foo"}, existingItems, map[string]string{}, jsonPath) + Convey("When there are itmes to remoe but none to add or update", func() { + p := createPatches([]string{"key-2", "key-3", "foo"}, existingItems, map[string]string{}, jsonPath, overwriteKeys) expected := []utils.JsonPatch{ utils.NewJsonPatch("remove", jsonPath, "key-2", ""), utils.NewJsonPatch("remove", jsonPath, "key-3", ""), @@ -525,9 +526,9 @@ func TestCreatePatches(t *testing.T) { So(sortJsonPatches(p), ShouldResemble, sortJsonPatches(expected)) }) - Convey("When when there are no itmes to remove but new items to add", func() { + Convey("When there are no itmes to remove but new items to add", func() { newItems := map[string]string{"new-key": "new-val", "key-1": "new-1"} - p := createPatches([]string{"key-1"}, existingItems, newItems, jsonPath) + p := createPatches([]string{"key-1"}, existingItems, newItems, jsonPath, overwriteKeys) expected := []utils.JsonPatch{ utils.NewJsonPatch("add", jsonPath, "new-key", newItems["new-key"]), utils.NewJsonPatch("replace", jsonPath, "key-1", newItems["key-1"]), @@ -535,9 +536,9 @@ func TestCreatePatches(t *testing.T) { So(sortJsonPatches(p), ShouldResemble, sortJsonPatches(expected)) }) - Convey("When when there are items to remove add and update", func() { + Convey("When there are items to remove add and update", func() { newItems := map[string]string{"new-key": "new-val", "key-2": "new-2", "key-4": "val-4"} - p := createPatches([]string{"key-1", "key-2", "key-3", "foo"}, existingItems, newItems, jsonPath) + p := createPatches([]string{"key-1", "key-2", "key-3", "foo"}, existingItems, newItems, jsonPath, overwriteKeys) expected := []utils.JsonPatch{ utils.NewJsonPatch("add", jsonPath, "new-key", newItems["new-key"]), utils.NewJsonPatch("add", jsonPath, "key-4", newItems["key-4"]), @@ -547,6 +548,16 @@ func TestCreatePatches(t *testing.T) { } So(sortJsonPatches(p), ShouldResemble, sortJsonPatches(expected)) }) + + Convey("When overwrite of keys is denied and there is already an existant key", func() { + overwriteKeys = false + newItems := map[string]string{"key-1": "new-2", "key-4": "val-4"} + p := createPatches([]string{}, existingItems, newItems, jsonPath, overwriteKeys) + expected := []utils.JsonPatch{ + utils.NewJsonPatch("add", jsonPath, "key-4", newItems["key-4"]), + } + So(sortJsonPatches(p), ShouldResemble, sortJsonPatches(expected)) + }) }) } @@ -896,3 +907,60 @@ func TestGetDynamicValue(t *testing.T) { }) } } + +func TestFilterTaints(t *testing.T) { + testcases := []struct { + name string + taints []corev1.Taint + maxTaints int + expectedResult []corev1.Taint + }{ + { + name: "no restriction on the number of taints", + taints: []corev1.Taint{ + { + Key: "feature.node.kubernetes.io/key1", + Value: "dummy", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + maxTaints: 0, + expectedResult: []corev1.Taint{ + { + Key: "feature.node.kubernetes.io/key1", + Value: "dummy", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + { + name: "max of 1 Taint should be generated", + taints: []corev1.Taint{ + { + Key: "feature.node.kubernetes.io/key1", + Value: "dummy", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "feature.node.kubernetes.io/key2", + Value: "dummy", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + maxTaints: 1, + expectedResult: []corev1.Taint{}, + }, + } + + mockMaster := newFakeMaster(nil) + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + mockMaster.config.Restrictions.MaxTaintsPerCR = tc.maxTaints + res := mockMaster.filterTaints(tc.taints) + Convey("The expected number of taints should be correct", t, func() { + So(len(res), ShouldEqual, len(tc.expectedResult)) + }) + }) + } +} diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index e34bb3e5f..b95685b3b 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -75,6 +75,16 @@ type ExtendedResources map[string]string // Annotations are used for NFD-related node metadata type Annotations map[string]string +// Restrictions contains the restrictions on the NF and NFR Crs +type Restrictions struct { + AllowedNamespaces *metav1.LabelSelector + MaxLabelsPerCR int + MaxTaintsPerCR int + MaxExtendedResourcesPerCR int + DenyNodeFeatureLabels bool + OverwriteLabels bool +} + // NFDConfig contains the configuration settings of NfdMaster. type NFDConfig struct { AutoDefaultNs bool @@ -88,6 +98,7 @@ type NFDConfig struct { LeaderElection LeaderElectionConfig NfdApiParallelism int Klog klogutils.KlogConfigOpts + Restrictions Restrictions } // LeaderElectionConfig contains the configuration for leader election @@ -273,6 +284,13 @@ func newDefaultConfig() *NFDConfig { RenewDeadline: utils.DurationVal{Duration: time.Duration(10) * time.Second}, }, Klog: make(map[string]string), + Restrictions: Restrictions{ + MaxLabelsPerCR: 0, + MaxTaintsPerCR: 0, + MaxExtendedResourcesPerCR: 0, + OverwriteLabels: true, + DenyNodeFeatureLabels: false, + }, } } @@ -620,7 +638,7 @@ func (m *nfdMaster) updateMasterNode() error { p := createPatches([]string{m.instanceAnnotation(nfdv1alpha1.MasterVersionAnnotation)}, node.Annotations, nil, - "/metadata/annotations") + "/metadata/annotations", true) err = patchNode(m.k8sClient, node.Name, p) if err != nil { @@ -661,6 +679,11 @@ func (m *nfdMaster) filterFeatureLabels(labels Labels, features *nfdv1alpha1.Fea } } + if m.config.Restrictions.MaxLabelsPerCR > 0 && len(outLabels) > m.config.Restrictions.MaxLabelsPerCR { + klog.InfoS("number of labels in the request exceeds the restriction maximum labels per CR, skipping") + outLabels = Labels{} + } + return outLabels, extendedResources } @@ -715,7 +738,7 @@ func getDynamicValue(value string, features *nfdv1alpha1.Features) (string, erro return element, nil } -func filterTaints(taints []corev1.Taint) []corev1.Taint { +func (m *nfdMaster) filterTaints(taints []corev1.Taint) []corev1.Taint { outTaints := []corev1.Taint{} for _, taint := range taints { @@ -726,6 +749,12 @@ func filterTaints(taints []corev1.Taint) []corev1.Taint { outTaints = append(outTaints, taint) } } + + if m.config.Restrictions.MaxTaintsPerCR > 0 && len(taints) > m.config.Restrictions.MaxTaintsPerCR { + klog.InfoS("number of taints in the request exceeds the restriction maximum taints per CR, skipping") + outTaints = []corev1.Taint{} + } + return outTaints } @@ -1031,13 +1060,24 @@ func (m *nfdMaster) refreshNodeFeatures(cli k8sclient.Interface, node *corev1.No maps.Copy(extendedResources, crExtendedResources) extendedResources = m.filterExtendedResources(features, extendedResources) + if m.config.Restrictions.MaxExtendedResourcesPerCR > 0 && len(extendedResources) > m.config.Restrictions.MaxExtendedResourcesPerCR { + klog.InfoS("number of extended resources in the request exceeds the restriction maximum extended resources per CR, skipping") + extendedResources = map[string]string{} + } + // Annotations annotations := m.filterFeatureAnnotations(crAnnotations) // Taints var taints []corev1.Taint if m.config.EnableTaints { - taints = filterTaints(crTaints) + taints = m.filterTaints(crTaints) + } + + // If we deny node feature labels, we'll empty the labels variable + if m.config.Restrictions.DenyNodeFeatureLabels { + klog.InfoS("node feature labels are denied, skipping...") + labels = map[string]string{} } if m.config.NoPublish { @@ -1114,7 +1154,7 @@ func setTaints(cli k8sclient.Interface, taints []corev1.Taint, node *corev1.Node newAnnotations[nfdv1alpha1.NodeTaintsAnnotation] = strings.Join(taintStrs, ",") } - patches := createPatches([]string{nfdv1alpha1.NodeTaintsAnnotation}, node.Annotations, newAnnotations, "/metadata/annotations") + patches := createPatches([]string{nfdv1alpha1.NodeTaintsAnnotation}, node.Annotations, newAnnotations, "/metadata/annotations", true) if len(patches) > 0 { if err := patchNode(cli, node.Name, patches); err != nil { return fmt.Errorf("error while patching node object: %w", err) @@ -1254,7 +1294,7 @@ func (m *nfdMaster) updateNodeObject(cli k8sclient.Interface, node *corev1.Node, // Create JSON patches for changes in labels and annotations oldLabels := stringToNsNames(node.Annotations[m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation)], nfdv1alpha1.FeatureLabelNs) oldAnnotations := stringToNsNames(node.Annotations[m.instanceAnnotation(nfdv1alpha1.FeatureAnnotationsTrackingAnnotation)], nfdv1alpha1.FeatureAnnotationNs) - patches := createPatches(oldLabels, node.Labels, labels, "/metadata/labels") + patches := createPatches(oldLabels, node.Labels, labels, "/metadata/labels", m.config.Restrictions.OverwriteLabels) oldAnnotations = append(oldAnnotations, []string{ m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation), m.instanceAnnotation(nfdv1alpha1.ExtendedResourceAnnotation), @@ -1262,7 +1302,7 @@ func (m *nfdMaster) updateNodeObject(cli k8sclient.Interface, node *corev1.Node, // Clean up deprecated/stale nfd version annotations m.instanceAnnotation(nfdv1alpha1.MasterVersionAnnotation), m.instanceAnnotation(nfdv1alpha1.WorkerVersionAnnotation)}...) - patches = append(patches, createPatches(oldAnnotations, node.Annotations, annotations, "/metadata/annotations")...) + patches = append(patches, createPatches(oldAnnotations, node.Annotations, annotations, "/metadata/annotations", true)...) // patch node status with extended resource changes statusPatches := m.createExtendedResourcePatches(node, extendedResources) @@ -1294,7 +1334,7 @@ func (m *nfdMaster) updateNodeObject(cli k8sclient.Interface, node *corev1.Node, } // createPatches is a generic helper that returns json patch operations to perform -func createPatches(removeKeys []string, oldItems map[string]string, newItems map[string]string, jsonPath string) []utils.JsonPatch { +func createPatches(removeKeys []string, oldItems map[string]string, newItems map[string]string, jsonPath string, overwrite bool) []utils.JsonPatch { patches := []utils.JsonPatch{} // Determine items to remove @@ -1309,7 +1349,7 @@ func createPatches(removeKeys []string, oldItems map[string]string, newItems map // Determine items to add or replace for key, newVal := range newItems { if oldVal, ok := oldItems[key]; ok { - if newVal != oldVal { + if newVal != oldVal && overwrite { patches = append(patches, utils.NewJsonPatch("replace", jsonPath, key, newVal)) } } else { @@ -1511,8 +1551,10 @@ func (m *nfdMaster) startNfdApiController() error { } klog.InfoS("starting the nfd api controller") m.nfdController, err = newNfdController(kubeconfig, nfdApiControllerOptions{ - DisableNodeFeature: !nfdfeatures.NFDFeatureGate.Enabled(nfdfeatures.NodeFeatureAPI), - ResyncPeriod: m.config.ResyncPeriod.Duration, + DisableNodeFeature: !nfdfeatures.NFDFeatureGate.Enabled(nfdfeatures.NodeFeatureAPI), + ResyncPeriod: m.config.ResyncPeriod.Duration, + K8sClient: m.k8sClient, + NodeFeatureNamespaceSelector: m.config.Restrictions.AllowedNamespaces, }) if err != nil { return fmt.Errorf("failed to initialize CRD controller: %w", err) diff --git a/test/e2e/node_feature_discovery_test.go b/test/e2e/node_feature_discovery_test.go index 38e0a49f0..dd692ce76 100644 --- a/test/e2e/node_feature_discovery_test.go +++ b/test/e2e/node_feature_discovery_test.go @@ -853,6 +853,227 @@ core: } eventuallyNonControlPlaneNodes(ctx, f.ClientSet).WithTimeout(1 * time.Minute).Should(MatchLabels(expectedLabels, nodes)) }) + + Context("allowed namespaces restriction is respected or not", func() { + BeforeEach(func(ctx context.Context) { + extraMasterPodSpecOpts = []testpod.SpecOption{ + testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"), + } + cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", ` +restrictions: + allowedNamespaces: + matchLabels: + kubernetes.io/metadata.name: "fake" +`) + _, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) + It("Nothing should be created", func(ctx context.Context) { + // deploy node feature object + if !useNodeFeatureApi { + Skip("NodeFeature API not enabled") + } + + nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet) + Expect(err).NotTo(HaveOccurred()) + + targetNodeName := nodes[0].Name + Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found") + + // Apply Node Feature object + By("Creating NodeFeature object") + nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(ctx, nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying node labels from NodeFeature object #1 are not created") + // No labels should be created since the f.Namespace is not in the allowed Namespaces + expectedLabels := map[string]k8sLabels{} + eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes)) + + By("Deleting NodeFeature object") + err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("max labels, taints restrictions should be respected", func() { + BeforeEach(func(ctx context.Context) { + extraMasterPodSpecOpts = []testpod.SpecOption{ + testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"), + } + cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", ` +enableTaints: true +restrictions: + maxLabelsPerCR: 1 + maxTaintsPerCR: 1 +`) + _, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) + It("No labels or taints should be created", func(ctx context.Context) { + nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet) + Expect(err).NotTo(HaveOccurred()) + + targetNodeName := nodes[0].Name + Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found") + + By("Creating nfd-worker config") + cm := testutils.NewConfigMap("nfd-worker-conf", "nfd-worker.conf", ` +core: + sleepInterval: "1s" + featureSources: ["fake"] + labelSources: [] +`) + cm, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + By("Creating nfd-worker daemonset") + podSpecOpts := createPodSpecOpts( + testpod.SpecWithContainerImage(dockerImage()), + testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), + ) + workerDS := testds.NFDWorker(podSpecOpts...) + workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(ctx, workerDS, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for worker daemonset pods to be ready") + Expect(testpod.WaitForReady(ctx, f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 2)).NotTo(HaveOccurred()) + + // Add features from NodeFeatureRule #3 + By("Creating NodeFeatureRules #3") + Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-3.yaml")).NotTo(HaveOccurred()) + + By("Verifying node taints and annotation from NodeFeatureRules #3") + expectedLabels := map[string]k8sLabels{} + + expectedTaints := map[string][]corev1.Taint{} + + eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes)) + eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes)) + + By("Deleting NodeFeatureRule #3") + err = nfdClient.NfdV1alpha1().NodeFeatureRules().Delete(ctx, "e2e-test-3", metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + By("Verifying taints from NodeFeatureRules #3 were removed") + expectedTaints["*"] = []corev1.Taint{} + eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes)) + }) + }) + + Context("max extended resources restriction should be respected", func() { + BeforeEach(func(ctx context.Context) { + extraMasterPodSpecOpts = []testpod.SpecOption{ + testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"), + } + cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", ` +restrictions: + maxExtendedResourcesPerCR: 1 +`) + _, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) + It("Nothing should be created", func(ctx context.Context) { + nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet) + Expect(err).NotTo(HaveOccurred()) + + targetNodeName := nodes[0].Name + Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found") + + By("Creating nfd-worker config") + cm := testutils.NewConfigMap("nfd-worker-conf", "nfd-worker.conf", ` +core: + sleepInterval: "1s" + featureSources: ["fake"] + labelSources: [] +`) + cm, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + By("Creating nfd-worker daemonset") + podSpecOpts := createPodSpecOpts( + testpod.SpecWithContainerImage(dockerImage()), + testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), + ) + workerDS := testds.NFDWorker(podSpecOpts...) + workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(ctx, workerDS, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for worker daemonset pods to be ready") + Expect(testpod.WaitForReady(ctx, f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 2)).NotTo(HaveOccurred()) + + expectedAnnotations := map[string]k8sAnnotations{ + "*": {}, + } + expectedCapacity := map[string]corev1.ResourceList{ + "*": {}, + } + + By("Creating NodeFeatureRules #4") + Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-4.yaml")).NotTo(HaveOccurred()) + + By("Verifying node annotations from NodeFeatureRules #4") + eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes)) + + By("Verifying node status capacity from NodeFeatureRules #4") + eventuallyNonControlPlaneNodes(ctx, f.ClientSet).WithTimeout(1 * time.Minute).Should(MatchCapacity(expectedCapacity, nodes)) + + By("Deleting NodeFeatureRules #4") + err = nfdClient.NfdV1alpha1().NodeFeatureRules().Delete(ctx, "e2e-extened-resource-test", metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Deleting nfd-worker daemonset") + err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(ctx, workerDS.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + if useNodeFeatureApi { + By("Verify that labels from nfd-worker are garbage-collected") + expectedLabels := map[string]k8sLabels{ + "*": {}, + } + eventuallyNonControlPlaneNodes(ctx, f.ClientSet).WithTimeout(1 * time.Minute).Should(MatchLabels(expectedLabels, nodes)) + } + }) + }) + + Context("deny node feature labels restriction should be respected", func() { + BeforeEach(func(ctx context.Context) { + extraMasterPodSpecOpts = []testpod.SpecOption{ + testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"), + } + cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", ` +restrictions: + denyNodeFeatureLabels: true +`) + _, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) + It("No labels should be created", func(ctx context.Context) { + // deploy node feature object + if !useNodeFeatureApi { + Skip("NodeFeature API not enabled") + } + + nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet) + Expect(err).NotTo(HaveOccurred()) + + targetNodeName := nodes[0].Name + Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found") + + // Apply Node Feature object + By("Creating NodeFeature object") + nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(ctx, nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying node labels from NodeFeature object #1 are not created") + + expectedLabels := map[string]k8sLabels{ + "*": {}, + } + eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes)) + + By("Deleting NodeFeature object") + err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) + }) }) // Test NodeFeatureGroups