Discover node features as annotations

Signed-off-by: Carlos Eduardo Arango Gutierrez <eduardoa@nvidia.com>
Co-authored-by: bebc <mchf1990212@gmail.com>
Co-authored-by: Markus Lehtonen <markus.lehtonen@intel.com>

deployment/base/nfd-crds/nfd-api-crds.yaml

@@ -153,6 +153,11 @@ spec:
                   description: Rule defines a rule for node customization such as
                     labeling.
                   properties:
+                    annotations:
+                      additionalProperties:
+                        type: string
+                      description: Annotations to create if the rule matches.
+                      type: object
                     extendedResources:
                       additionalProperties:
                         type: string

deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml

@@ -153,6 +153,11 @@ spec:
                   description: Rule defines a rule for node customization such as
                     labeling.
                   properties:
+                    annotations:
+                      additionalProperties:
+                        type: string
+                      description: Annotations to create if the rule matches.
+                      type: object
                     extendedResources:
                       additionalProperties:
                         type: string

docs/get-started/introduction.md

@@ -18,8 +18,8 @@ sort: 1
 This software enables node feature discovery for Kubernetes. It detects
 hardware features available on each node in a Kubernetes cluster, and
 advertises those features using node labels and optionally node extended
-resources and node taints. Node Feature Discovery is compatible with any recent
+resources, annotations and node taints. Node Feature Discovery is compatible
-version of Kubernetes (v1.21+).
+with any recent version of Kubernetes (v1.21+).
 
 NFD consists of four software components:
 

docs/usage/customization-guide.md

@@ -594,6 +594,54 @@ details.
 > labels specified in the `labels` field will override anything
 > originating from `labelsTemplate`.
 
+#### Node Annotations
+
+The `.annotations` field is a list of features to be advertised as annotations.
+
+Take this rule as a referential example:
+
+```yaml
+apiVersion: nfd.k8s-sigs.io/v1alpha1
+kind: NodeFeatureRule
+metadata:
+  name: feature-annotations-example
+spec:
+  rules:
+    - name: "annotation-example"
+      annotations:
+        defaul-ns-annotation: "foo"
+        feature.node.kubernetes.io/defaul-ns-annotation-2: "bar"
+        custom.vendor.io/feature: "baz"
+      matchFeatures:
+        - feature: kernel.version
+          matchExpressions:
+            major: {op: Exists}
+```
+
+This will yield into the following node annotations:
+
+```yaml
+  annotations:
+    ...
+    feature.node.kubernetes.io/defaul-ns-annotation: "foo"
+    feature.node.kubernetes.io/defaul-ns-annotation-2: "bar"
+    custom.vendor.io/feature: "baz"
+    ...
+```
+
+NFD enforces some limitations to the namespace (or prefix)/ of the annotations:
+
+- `kubernetes.io/` and its sub-namespaces (like `sub.ns.kubernetes.io/`) cannot
+  generally be used
+- the only exception is `feature.node.kubernetes.io/` and its sub-namespaces
+  (like `sub.ns.feature.node.kubernetes.io`)
+- unprefixed names will get prefixed with `feature.node.kubernetes.io/`
+  automatically (e.g. `foo` becomes `feature.node.kubernetes.io/foo`)
+
+> **NOTE:** The `annotations` field has will only advertise features via node
+> annotations the features won't be advertised as node labels unless they are
+> specified in the `labels` field.
+
 #### Taints
 
 *taints* is a list of taint entries and each entry can have `key`, `value` and `effect`,

pkg/apis/nfd/v1alpha1/annotations_labels.go

@@ -60,9 +60,18 @@ const (
 	// NodeTaintsAnnotation is the annotation that holds the taints that nfd-master set on the node
 	NodeTaintsAnnotation = AnnotationNs + "/taints"
 
+	// FeatureAnnotationsTrackingAnnotation is the annotation that holds all feature annotations that nfd-master set on the node
+	FeatureAnnotationsTrackingAnnotation = AnnotationNs + "/feature-annotations"
+
 	// NodeFeatureObjNodeNameLabel is the label that specifies which node the
 	// NodeFeature object is targeting. Creators of NodeFeature objects must
 	// set this label and consumers of the objects are supposed to use the
 	// label for filtering features designated for a certain node.
 	NodeFeatureObjNodeNameLabel = "nfd.node.kubernetes.io/node-name"
+
+	// FeatureAnnotationNs is the (default) namespace for feature annotations.
+	FeatureAnnotationNs = "feature.node.kubernetes.io"
+
+	// FeatureAnnotationSubNsSuffix is the suffix for allowed feature annotation sub-namespaces.
+	FeatureAnnotationSubNsSuffix = "." + FeatureAnnotationNs
 )

pkg/apis/nfd/v1alpha1/rule.go

@@ -24,6 +24,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/klog/v2"
+
 	"sigs.k8s.io/node-feature-discovery/pkg/utils"
 )
 
@@ -32,6 +33,7 @@ import (
 type RuleOutput struct {
 	ExtendedResources map[string]string
 	Labels            map[string]string
+	Annotations       map[string]string
 	Vars              map[string]string
 	Taints            []corev1.Taint
 }
@@ -101,7 +103,7 @@ func (r *Rule) Execute(features *Features) (RuleOutput, error) {
 		vars[k] = v
 	}
 
-	ret := RuleOutput{ExtendedResources: extendedResources, Labels: labels, Vars: vars, Taints: r.Taints}
+	ret := RuleOutput{ExtendedResources: extendedResources, Labels: labels, Vars: vars, Taints: r.Taints, Annotations: r.Annotations}
 	klog.V(2).InfoS("rule matched", "ruleName", r.Name, "ruleOutput", utils.DelayedDumper(ret))
 	return ret, nil
 }

pkg/apis/nfd/v1alpha1/types.go

@@ -146,6 +146,10 @@ type Rule struct {
 	// +optional
 	LabelsTemplate string `json:"labelsTemplate"`
 
+	// Annotations to create if the rule matches.
+	// +optional
+	Annotations map[string]string `json:"annotations"`
+
 	// Vars is the variables to store if the rule matches. Variables do not
 	// directly inflict any changes in the node object. However, they can be
 	// referenced from other rules enabling more complex rule hierarchies,

pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go

@@ -513,6 +513,13 @@ func (in *Rule) DeepCopyInto(out *Rule) {
 			(*out)[key] = val
 		}
 	}
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.Vars != nil {
 		in, out := &in.Vars, &out.Vars
 		*out = make(map[string]string, len(*in))

pkg/nfd-master/nfd-master-internal_test.go

@@ -138,6 +138,12 @@ func TestUpdateNodeObject(t *testing.T) {
 		}
 		sort.Strings(fakeFeatureLabelNames)
 
+		fakeFeatureAnnotationsNames := make([]string, 0, len(fakeFeatureLabels))
+		for k := range fakeAnnotations {
+			fakeFeatureAnnotationsNames = append(fakeFeatureAnnotationsNames, strings.TrimPrefix(k, nfdv1alpha1.FeatureAnnotationNs+"/"))
+		}
+		sort.Strings(fakeFeatureAnnotationsNames)
+
 		fakeExtResourceNames := make([]string, 0, len(fakeExtResources))
 		for k := range fakeExtResources {
 			fakeExtResourceNames = append(fakeExtResourceNames, strings.TrimPrefix(k, nfdv1alpha1.FeatureLabelNs+"/"))
@@ -162,6 +168,7 @@ func TestUpdateNodeObject(t *testing.T) {
 			// Create a list of expected node metadata patches
 			metadataPatches := []apihelper.JsonPatch{
 				apihelper.NewJsonPatch("replace", "/metadata/annotations", nfdv1alpha1.AnnotationNs+"/feature-labels", strings.Join(fakeFeatureLabelNames, ",")),
+				apihelper.NewJsonPatch("add", "/metadata/annotations", nfdv1alpha1.FeatureAnnotationsTrackingAnnotation, strings.Join(fakeFeatureAnnotationsNames, ",")),
 				apihelper.NewJsonPatch("add", "/metadata/annotations", nfdv1alpha1.AnnotationNs+"/extended-resources", strings.Join(fakeExtResourceNames, ",")),
 				apihelper.NewJsonPatch("remove", "/metadata/labels", nfdv1alpha1.FeatureLabelNs+"/old-feature", ""),
 			}
@@ -176,7 +183,7 @@ func TestUpdateNodeObject(t *testing.T) {
 			mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(mockNode, nil).Twice()
 			mockAPIHelper.On("PatchNodeStatus", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(statusPatches))).Return(nil)
 			mockAPIHelper.On("PatchNode", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(metadataPatches))).Return(nil)
-			err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
+			err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
 
 			Convey("Error is nil", func() {
 				So(err, ShouldBeNil)
@@ -186,7 +193,7 @@ func TestUpdateNodeObject(t *testing.T) {
 		Convey("When I fail to update the node with feature labels", func() {
 			expectedError := fmt.Errorf("no client is passed, client:  <nil>")
 			mockAPIHelper.On("GetClient").Return(nil, expectedError)
-			err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
+			err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
 
 			Convey("Error is produced", func() {
 				So(err, ShouldResemble, expectedError)
@@ -196,7 +203,7 @@ func TestUpdateNodeObject(t *testing.T) {
 		Convey("When I fail to get a mock client while updating feature labels", func() {
 			expectedError := fmt.Errorf("no client is passed, client:  <nil>")
 			mockAPIHelper.On("GetClient").Return(nil, expectedError)
-			err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
+			err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
 
 			Convey("Error is produced", func() {
 				So(err, ShouldResemble, expectedError)
@@ -207,7 +214,7 @@ func TestUpdateNodeObject(t *testing.T) {
 			expectedError := errors.New("fake error")
 			mockAPIHelper.On("GetClient").Return(mockClient, nil)
 			mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(nil, expectedError).Twice()
-			err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
+			err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
 
 			Convey("Error is produced", func() {
 				So(err, ShouldEqual, expectedError)
@@ -220,7 +227,7 @@ func TestUpdateNodeObject(t *testing.T) {
 			mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(mockNode, nil).Twice()
 			mockAPIHelper.On("PatchNodeStatus", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(statusPatches))).Return(nil)
 			mockAPIHelper.On("PatchNode", mockClient, mockNodeName, mock.Anything).Return(expectedError).Twice()
-			err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
+			err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
 
 			Convey("Error is produced", func() {
 				So(err.Error(), ShouldEndWith, expectedError.Error())

pkg/nfd-master/nfd-master.go

@@ -465,7 +465,7 @@ func (m *nfdMaster) prune() error {
 		klog.InfoS("pruning node...", "nodeName", node.Name)
 
 		// Prune labels and extended resources
-		err := m.updateNodeObject(cli, node.Name, Labels{}, Annotations{}, ExtendedResources{}, []corev1.Taint{})
+		err := m.updateNodeObject(cli, node.Name, Labels{}, Annotations{}, Annotations{}, ExtendedResources{}, []corev1.Taint{})
 		if err != nil {
 			nodeUpdateFailures.Inc()
 			return fmt.Errorf("failed to prune node %q: %v", node.Name, err)
@@ -837,12 +837,12 @@ func filterExtendedResource(name, value string, features *nfdv1alpha1.Features)
 	return q.String(), nil
 }
 
-func (m *nfdMaster) refreshNodeFeatures(cli *kubernetes.Clientset, nodeName string, annotations Annotations, labels map[string]string, features *nfdv1alpha1.Features) error {
+func (m *nfdMaster) refreshNodeFeatures(cli *kubernetes.Clientset, nodeName string, nfdAnnotations Annotations, labels map[string]string, features *nfdv1alpha1.Features) error {
 	if labels == nil {
 		labels = make(map[string]string)
 	}
 
-	crLabels, crExtendedResources, crTaints := m.processNodeFeatureRule(nodeName, features)
+	crLabels, crAnnotations, crExtendedResources, crTaints := m.processNodeFeatureRule(nodeName, features)
 
 	// Mix in CR-originated labels
 	for k, v := range crLabels {
@@ -859,12 +859,16 @@ func (m *nfdMaster) refreshNodeFeatures(cli *kubernetes.Clientset, nodeName stri
 	}
 	extendedResources = filterExtendedResources(features, extendedResources)
 
+	// Annotations
+	featureAnnotations := m.filterFeatureAnnotations(crAnnotations)
+
+	// Taints
 	var taints []corev1.Taint
 	if m.config.EnableTaints {
 		taints = filterTaints(crTaints)
 	}
 
-	err := m.updateNodeObject(cli, nodeName, labels, annotations, extendedResources, taints)
+	err := m.updateNodeObject(cli, nodeName, labels, nfdAnnotations, featureAnnotations, extendedResources, taints)
 	if err != nil {
 		klog.ErrorS(err, "failed to update node", "nodeName", nodeName)
 		return err
@@ -974,13 +978,14 @@ func authorizeClient(c context.Context, checkNodeName bool, nodeName string) err
 	return nil
 }
 
-func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha1.Features) (Labels, ExtendedResources, []corev1.Taint) {
+func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha1.Features) (Labels, Annotations, ExtendedResources, []corev1.Taint) {
 	if m.nfdController == nil {
-		return nil, nil, nil
+		return nil, nil, nil, nil
 	}
 
 	extendedResources := ExtendedResources{}
 	labels := make(map[string]string)
+	annotations := make(map[string]string)
 	var taints []corev1.Taint
 	ruleSpecs, err := m.nfdController.ruleLister.List(k8sLabels.Everything())
 	sort.Slice(ruleSpecs, func(i, j int) bool {
@@ -989,7 +994,7 @@ func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha
 
 	if err != nil {
 		klog.ErrorS(err, "failed to list NodeFeatureRule resources")
-		return nil, nil, nil
+		return nil, nil, nil, nil
 	}
 
 	// Process all rule CRs
@@ -1016,6 +1021,9 @@ func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha
 			for k, v := range ruleOut.ExtendedResources {
 				extendedResources[k] = v
 			}
+			for k, v := range ruleOut.Annotations {
+				annotations[k] = v
+			}
 
 			// Feed back rule output to features map for subsequent rules to match
 			features.InsertAttributeFeatures(nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Labels)
@@ -1026,13 +1034,13 @@ func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha
 	processingTime := time.Since(processStart)
 	klog.V(2).InfoS("processed NodeFeatureRule objects", "nodeName", nodeName, "objectCount", len(ruleSpecs), "duration", processingTime)
 
-	return labels, extendedResources, taints
+	return labels, annotations, extendedResources, taints
 }
 
 // updateNodeObject ensures the Kubernetes node object is up to date,
 // creating new labels and extended resources where necessary and removing
 // outdated ones. Also updates the corresponding annotations.
-func (m *nfdMaster) updateNodeObject(cli *kubernetes.Clientset, nodeName string, labels Labels, annotations Annotations, extendedResources ExtendedResources, taints []corev1.Taint) error {
+func (m *nfdMaster) updateNodeObject(cli *kubernetes.Clientset, nodeName string, labels Labels, nfdAnnotations, featureAnnotations Annotations, extendedResources ExtendedResources, taints []corev1.Taint) error {
 	if cli == nil {
 		return fmt.Errorf("no client is passed, client:  %v", cli)
 	}
@@ -1051,7 +1059,7 @@ func (m *nfdMaster) updateNodeObject(cli *kubernetes.Clientset, nodeName string,
 			labelKeys = append(labelKeys, strings.TrimPrefix(key, nfdv1alpha1.FeatureLabelNs+"/"))
 		}
 		sort.Strings(labelKeys)
-		annotations[m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation)] = strings.Join(labelKeys, ",")
+		nfdAnnotations[m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation)] = strings.Join(labelKeys, ",")
 	}
 
 	// Store names of extended resources in an annotation
@@ -1062,23 +1070,43 @@ func (m *nfdMaster) updateNodeObject(cli *kubernetes.Clientset, nodeName string,
 			extendedResourceKeys = append(extendedResourceKeys, strings.TrimPrefix(key, nfdv1alpha1.FeatureLabelNs+"/"))
 		}
 		sort.Strings(extendedResourceKeys)
-		annotations[m.instanceAnnotation(nfdv1alpha1.ExtendedResourceAnnotation)] = strings.Join(extendedResourceKeys, ",")
+		nfdAnnotations[m.instanceAnnotation(nfdv1alpha1.ExtendedResourceAnnotation)] = strings.Join(extendedResourceKeys, ",")
+	}
+
+	// Store feature annotations
+	annotations := make(Annotations)
+	if len(featureAnnotations) > 0 {
+		// Store names of feature annotations in an annotation
+		annotationKeys := make([]string, 0, len(featureAnnotations))
+		for key := range featureAnnotations {
+			// Drop the ns part for annotations in the default ns
+			annotationKeys = append(annotationKeys, strings.TrimPrefix(key, nfdv1alpha1.FeatureAnnotationNs+"/"))
+		}
+		sort.Strings(annotationKeys)
+		nfdAnnotations[m.instanceAnnotation(nfdv1alpha1.FeatureAnnotationsTrackingAnnotation)] = strings.Join(annotationKeys, ",")
+		for k, v := range featureAnnotations {
+			annotations[k] = v
+		}
+	}
+
+	if len(nfdAnnotations) > 0 {
+		for k, v := range nfdAnnotations {
+			annotations[k] = v
+		}
 	}
 
 	// Create JSON patches for changes in labels and annotations
 	oldLabels := stringToNsNames(node.Annotations[m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation)], nfdv1alpha1.FeatureLabelNs)
+	oldAnnotations := stringToNsNames(node.Annotations[m.instanceAnnotation(nfdv1alpha1.FeatureAnnotationsTrackingAnnotation)], nfdv1alpha1.FeatureAnnotationNs)
 	patches := createPatches(oldLabels, node.Labels, labels, "/metadata/labels")
-	patches = append(patches,
+	oldAnnotations = append(oldAnnotations, []string{
-		createPatches(
-			[]string{
 		m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation),
 		m.instanceAnnotation(nfdv1alpha1.ExtendedResourceAnnotation),
+		m.instanceAnnotation(nfdv1alpha1.FeatureAnnotationsTrackingAnnotation),
 		// Clean up deprecated/stale nfd version annotations
 		m.instanceAnnotation(nfdv1alpha1.MasterVersionAnnotation),
-				m.instanceAnnotation(nfdv1alpha1.WorkerVersionAnnotation)},
+		m.instanceAnnotation(nfdv1alpha1.WorkerVersionAnnotation)}...)
-			node.Annotations,
+	patches = append(patches, createPatches(oldAnnotations, node.Annotations, annotations, "/metadata/annotations")...)
-			annotations,
-			"/metadata/annotations")...)
 
 	// patch node status with extended resource changes
 	statusPatches := m.createExtendedResourcePatches(node, extendedResources)
@@ -1379,3 +1407,27 @@ func (m *nfdMaster) nfdAPIUpdateHandlerWithLeaderElection() {
 
 	leaderElector.Run(ctx)
 }
+
+// Filter annotations by namespace. i.e. adds the possibly missing default namespace for annotations
+func (m *nfdMaster) filterFeatureAnnotations(annotations map[string]string) map[string]string {
+	outAnnotations := make(map[string]string)
+
+	for annotation, value := range annotations {
+		// Add possibly missing default ns
+		annotation := addNs(annotation, nfdv1alpha1.FeatureAnnotationNs)
+
+		ns, _ := splitNs(annotation)
+
+		// Check annotation namespace, filter out if ns is not whitelisted
+		if ns != nfdv1alpha1.FeatureAnnotationNs && !strings.HasSuffix(ns, nfdv1alpha1.FeatureAnnotationSubNsSuffix) {
+			// If the namespace is denied, and not present in the extraLabelNs, label will be ignored
+			if ns == "kubernetes.io" || strings.HasSuffix(ns, ".kubernetes.io") || ns == nfdv1alpha1.AnnotationNs {
+				klog.ErrorS(fmt.Errorf("namespace %v is not allowed", ns), fmt.Sprintf("Ignoring annotation %v\n", annotation))
+				continue
+			}
+		}
+
+		outAnnotations[annotation] = value
+	}
+	return outAnnotations
+}

test/e2e/data/nodefeaturerule-5.yaml

@@ -0,0 +1,19 @@
+apiVersion: nfd.k8s-sigs.io/v1alpha1
+kind: NodeFeatureRule
+metadata:
+  name: e2e-feature-annotations-test
+spec:
+  rules:
+    # Positive test expected to set the annotations
+    - name: "e2e-annotation-test"
+      annotations:
+        defaul-ns-annotation: "foo"
+        feature.node.kubernetes.io/defaul-ns-annotation-2: "bar"
+        custom.vendor.io/feature: "baz"
+        kubernetes.io/feature: "denied"
+        subns.kubernetes.io/blah: "denied"
+        nfd.node.kubernetes.io/xyz: "denied"
+      matchFeatures:
+        - feature: "fake.flag"
+          matchExpressions:
+            "flag_1": {op: Exists}

test/e2e/node_feature_discovery_test.go

@@ -803,6 +803,25 @@ core:
 					eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchCapacity(expectedCapacity, nodes))
 					eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))
 
+					By("Creating NodeFeatureRules #5")
+					Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-5.yaml")).NotTo(HaveOccurred())
+
+					By("Verifying node annotations from NodeFeatureRules #5")
+					expectedAnnotations["*"] = k8sAnnotations{
+						nfdv1alpha1.FeatureLabelNs + "/defaul-ns-annotation":   "foo",
+						nfdv1alpha1.FeatureLabelNs + "/defaul-ns-annotation-2": "bar",
+						"custom.vendor.example/feature":                        "baz",
+					}
+					eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))
+
+					By("Deleting NodeFeatureRule object")
+					err = nfdClient.NfdV1alpha1().NodeFeatureRules().Delete(ctx, " e2e-feature-annotations-test", metav1.DeleteOptions{})
+					Expect(err).NotTo(HaveOccurred())
+
+					By("Verifying node annotations from NodeFeatureRules #5 are deleted")
+					expectedAnnotations["*"] = k8sAnnotations{}
+					eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))
+
 					By("Deleting nfd-worker daemonset")
 					err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(ctx, workerDS.Name, metav1.DeleteOptions{})
 					Expect(err).NotTo(HaveOccurred())