diff --git a/deployment/base/nfd-crds/nfd-api-crds.yaml b/deployment/base/nfd-crds/nfd-api-crds.yaml index 6866c7ffe..5c94140bf 100644 --- a/deployment/base/nfd-crds/nfd-api-crds.yaml +++ b/deployment/base/nfd-crds/nfd-api-crds.yaml @@ -153,6 +153,11 @@ spec: description: Rule defines a rule for node customization such as labeling. properties: + annotations: + additionalProperties: + type: string + description: Annotations to create if the rule matches. + type: object extendedResources: additionalProperties: type: string diff --git a/deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml b/deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml index 6866c7ffe..5c94140bf 100644 --- a/deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml +++ b/deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml @@ -153,6 +153,11 @@ spec: description: Rule defines a rule for node customization such as labeling. properties: + annotations: + additionalProperties: + type: string + description: Annotations to create if the rule matches. + type: object extendedResources: additionalProperties: type: string diff --git a/docs/get-started/introduction.md b/docs/get-started/introduction.md index 34f57fea9..4e2754bb2 100644 --- a/docs/get-started/introduction.md +++ b/docs/get-started/introduction.md @@ -18,8 +18,8 @@ sort: 1 This software enables node feature discovery for Kubernetes. It detects hardware features available on each node in a Kubernetes cluster, and advertises those features using node labels and optionally node extended -resources and node taints. Node Feature Discovery is compatible with any recent -version of Kubernetes (v1.21+). +resources, annotations and node taints. Node Feature Discovery is compatible +with any recent version of Kubernetes (v1.21+). NFD consists of four software components: diff --git a/docs/usage/customization-guide.md b/docs/usage/customization-guide.md index c5d7676fc..1efda0121 100644 --- a/docs/usage/customization-guide.md +++ b/docs/usage/customization-guide.md @@ -594,6 +594,54 @@ details. > labels specified in the `labels` field will override anything > originating from `labelsTemplate`. +#### Node Annotations + +The `.annotations` field is a list of features to be advertised as annotations. + +Take this rule as a referential example: + +```yaml +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: feature-annotations-example +spec: + rules: + - name: "annotation-example" + annotations: + defaul-ns-annotation: "foo" + feature.node.kubernetes.io/defaul-ns-annotation-2: "bar" + custom.vendor.io/feature: "baz" + matchFeatures: + - feature: kernel.version + matchExpressions: + major: {op: Exists} +``` + +This will yield into the following node annotations: + +```yaml + annotations: + ... + feature.node.kubernetes.io/defaul-ns-annotation: "foo" + feature.node.kubernetes.io/defaul-ns-annotation-2: "bar" + custom.vendor.io/feature: "baz" + ... +``` + +NFD enforces some limitations to the namespace (or prefix)/ of the annotations: + +- `kubernetes.io/` and its sub-namespaces (like `sub.ns.kubernetes.io/`) cannot + generally be used +- the only exception is `feature.node.kubernetes.io/` and its sub-namespaces + (like `sub.ns.feature.node.kubernetes.io`) +- unprefixed names will get prefixed with `feature.node.kubernetes.io/` + automatically (e.g. `foo` becomes `feature.node.kubernetes.io/foo`) + +> **NOTE:** The `annotations` field has will only advertise features via node +> annotations the features won't be advertised as node labels unless they are +> specified in the `labels` field. + #### Taints *taints* is a list of taint entries and each entry can have `key`, `value` and `effect`, diff --git a/pkg/apis/nfd/v1alpha1/annotations_labels.go b/pkg/apis/nfd/v1alpha1/annotations_labels.go index 8eda7c1cc..b1f848245 100644 --- a/pkg/apis/nfd/v1alpha1/annotations_labels.go +++ b/pkg/apis/nfd/v1alpha1/annotations_labels.go @@ -60,9 +60,18 @@ const ( // NodeTaintsAnnotation is the annotation that holds the taints that nfd-master set on the node NodeTaintsAnnotation = AnnotationNs + "/taints" + // FeatureAnnotationsTrackingAnnotation is the annotation that holds all feature annotations that nfd-master set on the node + FeatureAnnotationsTrackingAnnotation = AnnotationNs + "/feature-annotations" + // NodeFeatureObjNodeNameLabel is the label that specifies which node the // NodeFeature object is targeting. Creators of NodeFeature objects must // set this label and consumers of the objects are supposed to use the // label for filtering features designated for a certain node. NodeFeatureObjNodeNameLabel = "nfd.node.kubernetes.io/node-name" + + // FeatureAnnotationNs is the (default) namespace for feature annotations. + FeatureAnnotationNs = "feature.node.kubernetes.io" + + // FeatureAnnotationSubNsSuffix is the suffix for allowed feature annotation sub-namespaces. + FeatureAnnotationSubNsSuffix = "." + FeatureAnnotationNs ) diff --git a/pkg/apis/nfd/v1alpha1/rule.go b/pkg/apis/nfd/v1alpha1/rule.go index 6bea00dee..53b9ad188 100644 --- a/pkg/apis/nfd/v1alpha1/rule.go +++ b/pkg/apis/nfd/v1alpha1/rule.go @@ -24,6 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/klog/v2" + "sigs.k8s.io/node-feature-discovery/pkg/utils" ) @@ -32,6 +33,7 @@ import ( type RuleOutput struct { ExtendedResources map[string]string Labels map[string]string + Annotations map[string]string Vars map[string]string Taints []corev1.Taint } @@ -101,7 +103,7 @@ func (r *Rule) Execute(features *Features) (RuleOutput, error) { vars[k] = v } - ret := RuleOutput{ExtendedResources: extendedResources, Labels: labels, Vars: vars, Taints: r.Taints} + ret := RuleOutput{ExtendedResources: extendedResources, Labels: labels, Vars: vars, Taints: r.Taints, Annotations: r.Annotations} klog.V(2).InfoS("rule matched", "ruleName", r.Name, "ruleOutput", utils.DelayedDumper(ret)) return ret, nil } diff --git a/pkg/apis/nfd/v1alpha1/types.go b/pkg/apis/nfd/v1alpha1/types.go index 21b5567df..10f069ad7 100644 --- a/pkg/apis/nfd/v1alpha1/types.go +++ b/pkg/apis/nfd/v1alpha1/types.go @@ -146,6 +146,10 @@ type Rule struct { // +optional LabelsTemplate string `json:"labelsTemplate"` + // Annotations to create if the rule matches. + // +optional + Annotations map[string]string `json:"annotations"` + // Vars is the variables to store if the rule matches. Variables do not // directly inflict any changes in the node object. However, they can be // referenced from other rules enabling more complex rule hierarchies, diff --git a/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go index fac8c2ad1..971600f9d 100644 --- a/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go @@ -513,6 +513,13 @@ func (in *Rule) DeepCopyInto(out *Rule) { (*out)[key] = val } } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Vars != nil { in, out := &in.Vars, &out.Vars *out = make(map[string]string, len(*in)) diff --git a/pkg/nfd-master/nfd-master-internal_test.go b/pkg/nfd-master/nfd-master-internal_test.go index e3fd265fe..0686a49ee 100644 --- a/pkg/nfd-master/nfd-master-internal_test.go +++ b/pkg/nfd-master/nfd-master-internal_test.go @@ -138,6 +138,12 @@ func TestUpdateNodeObject(t *testing.T) { } sort.Strings(fakeFeatureLabelNames) + fakeFeatureAnnotationsNames := make([]string, 0, len(fakeFeatureLabels)) + for k := range fakeAnnotations { + fakeFeatureAnnotationsNames = append(fakeFeatureAnnotationsNames, strings.TrimPrefix(k, nfdv1alpha1.FeatureAnnotationNs+"/")) + } + sort.Strings(fakeFeatureAnnotationsNames) + fakeExtResourceNames := make([]string, 0, len(fakeExtResources)) for k := range fakeExtResources { fakeExtResourceNames = append(fakeExtResourceNames, strings.TrimPrefix(k, nfdv1alpha1.FeatureLabelNs+"/")) @@ -162,6 +168,7 @@ func TestUpdateNodeObject(t *testing.T) { // Create a list of expected node metadata patches metadataPatches := []apihelper.JsonPatch{ apihelper.NewJsonPatch("replace", "/metadata/annotations", nfdv1alpha1.AnnotationNs+"/feature-labels", strings.Join(fakeFeatureLabelNames, ",")), + apihelper.NewJsonPatch("add", "/metadata/annotations", nfdv1alpha1.FeatureAnnotationsTrackingAnnotation, strings.Join(fakeFeatureAnnotationsNames, ",")), apihelper.NewJsonPatch("add", "/metadata/annotations", nfdv1alpha1.AnnotationNs+"/extended-resources", strings.Join(fakeExtResourceNames, ",")), apihelper.NewJsonPatch("remove", "/metadata/labels", nfdv1alpha1.FeatureLabelNs+"/old-feature", ""), } @@ -176,7 +183,7 @@ func TestUpdateNodeObject(t *testing.T) { mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(mockNode, nil).Twice() mockAPIHelper.On("PatchNodeStatus", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(statusPatches))).Return(nil) mockAPIHelper.On("PatchNode", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(metadataPatches))).Return(nil) - err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil) + err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil) Convey("Error is nil", func() { So(err, ShouldBeNil) @@ -186,7 +193,7 @@ func TestUpdateNodeObject(t *testing.T) { Convey("When I fail to update the node with feature labels", func() { expectedError := fmt.Errorf("no client is passed, client: ") mockAPIHelper.On("GetClient").Return(nil, expectedError) - err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil) + err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil) Convey("Error is produced", func() { So(err, ShouldResemble, expectedError) @@ -196,7 +203,7 @@ func TestUpdateNodeObject(t *testing.T) { Convey("When I fail to get a mock client while updating feature labels", func() { expectedError := fmt.Errorf("no client is passed, client: ") mockAPIHelper.On("GetClient").Return(nil, expectedError) - err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil) + err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil) Convey("Error is produced", func() { So(err, ShouldResemble, expectedError) @@ -207,7 +214,7 @@ func TestUpdateNodeObject(t *testing.T) { expectedError := errors.New("fake error") mockAPIHelper.On("GetClient").Return(mockClient, nil) mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(nil, expectedError).Twice() - err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil) + err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil) Convey("Error is produced", func() { So(err, ShouldEqual, expectedError) @@ -220,7 +227,7 @@ func TestUpdateNodeObject(t *testing.T) { mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(mockNode, nil).Twice() mockAPIHelper.On("PatchNodeStatus", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(statusPatches))).Return(nil) mockAPIHelper.On("PatchNode", mockClient, mockNodeName, mock.Anything).Return(expectedError).Twice() - err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil) + err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil) Convey("Error is produced", func() { So(err.Error(), ShouldEndWith, expectedError.Error()) diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index 0197c24d6..a6c9f9e3e 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -465,7 +465,7 @@ func (m *nfdMaster) prune() error { klog.InfoS("pruning node...", "nodeName", node.Name) // Prune labels and extended resources - err := m.updateNodeObject(cli, node.Name, Labels{}, Annotations{}, ExtendedResources{}, []corev1.Taint{}) + err := m.updateNodeObject(cli, node.Name, Labels{}, Annotations{}, Annotations{}, ExtendedResources{}, []corev1.Taint{}) if err != nil { nodeUpdateFailures.Inc() return fmt.Errorf("failed to prune node %q: %v", node.Name, err) @@ -837,12 +837,12 @@ func filterExtendedResource(name, value string, features *nfdv1alpha1.Features) return q.String(), nil } -func (m *nfdMaster) refreshNodeFeatures(cli *kubernetes.Clientset, nodeName string, annotations Annotations, labels map[string]string, features *nfdv1alpha1.Features) error { +func (m *nfdMaster) refreshNodeFeatures(cli *kubernetes.Clientset, nodeName string, nfdAnnotations Annotations, labels map[string]string, features *nfdv1alpha1.Features) error { if labels == nil { labels = make(map[string]string) } - crLabels, crExtendedResources, crTaints := m.processNodeFeatureRule(nodeName, features) + crLabels, crAnnotations, crExtendedResources, crTaints := m.processNodeFeatureRule(nodeName, features) // Mix in CR-originated labels for k, v := range crLabels { @@ -859,12 +859,16 @@ func (m *nfdMaster) refreshNodeFeatures(cli *kubernetes.Clientset, nodeName stri } extendedResources = filterExtendedResources(features, extendedResources) + // Annotations + featureAnnotations := m.filterFeatureAnnotations(crAnnotations) + + // Taints var taints []corev1.Taint if m.config.EnableTaints { taints = filterTaints(crTaints) } - err := m.updateNodeObject(cli, nodeName, labels, annotations, extendedResources, taints) + err := m.updateNodeObject(cli, nodeName, labels, nfdAnnotations, featureAnnotations, extendedResources, taints) if err != nil { klog.ErrorS(err, "failed to update node", "nodeName", nodeName) return err @@ -974,13 +978,14 @@ func authorizeClient(c context.Context, checkNodeName bool, nodeName string) err return nil } -func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha1.Features) (Labels, ExtendedResources, []corev1.Taint) { +func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha1.Features) (Labels, Annotations, ExtendedResources, []corev1.Taint) { if m.nfdController == nil { - return nil, nil, nil + return nil, nil, nil, nil } extendedResources := ExtendedResources{} labels := make(map[string]string) + annotations := make(map[string]string) var taints []corev1.Taint ruleSpecs, err := m.nfdController.ruleLister.List(k8sLabels.Everything()) sort.Slice(ruleSpecs, func(i, j int) bool { @@ -989,7 +994,7 @@ func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha if err != nil { klog.ErrorS(err, "failed to list NodeFeatureRule resources") - return nil, nil, nil + return nil, nil, nil, nil } // Process all rule CRs @@ -1016,6 +1021,9 @@ func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha for k, v := range ruleOut.ExtendedResources { extendedResources[k] = v } + for k, v := range ruleOut.Annotations { + annotations[k] = v + } // Feed back rule output to features map for subsequent rules to match features.InsertAttributeFeatures(nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Labels) @@ -1026,13 +1034,13 @@ func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha processingTime := time.Since(processStart) klog.V(2).InfoS("processed NodeFeatureRule objects", "nodeName", nodeName, "objectCount", len(ruleSpecs), "duration", processingTime) - return labels, extendedResources, taints + return labels, annotations, extendedResources, taints } // updateNodeObject ensures the Kubernetes node object is up to date, // creating new labels and extended resources where necessary and removing // outdated ones. Also updates the corresponding annotations. -func (m *nfdMaster) updateNodeObject(cli *kubernetes.Clientset, nodeName string, labels Labels, annotations Annotations, extendedResources ExtendedResources, taints []corev1.Taint) error { +func (m *nfdMaster) updateNodeObject(cli *kubernetes.Clientset, nodeName string, labels Labels, nfdAnnotations, featureAnnotations Annotations, extendedResources ExtendedResources, taints []corev1.Taint) error { if cli == nil { return fmt.Errorf("no client is passed, client: %v", cli) } @@ -1051,7 +1059,7 @@ func (m *nfdMaster) updateNodeObject(cli *kubernetes.Clientset, nodeName string, labelKeys = append(labelKeys, strings.TrimPrefix(key, nfdv1alpha1.FeatureLabelNs+"/")) } sort.Strings(labelKeys) - annotations[m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation)] = strings.Join(labelKeys, ",") + nfdAnnotations[m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation)] = strings.Join(labelKeys, ",") } // Store names of extended resources in an annotation @@ -1062,23 +1070,43 @@ func (m *nfdMaster) updateNodeObject(cli *kubernetes.Clientset, nodeName string, extendedResourceKeys = append(extendedResourceKeys, strings.TrimPrefix(key, nfdv1alpha1.FeatureLabelNs+"/")) } sort.Strings(extendedResourceKeys) - annotations[m.instanceAnnotation(nfdv1alpha1.ExtendedResourceAnnotation)] = strings.Join(extendedResourceKeys, ",") + nfdAnnotations[m.instanceAnnotation(nfdv1alpha1.ExtendedResourceAnnotation)] = strings.Join(extendedResourceKeys, ",") + } + + // Store feature annotations + annotations := make(Annotations) + if len(featureAnnotations) > 0 { + // Store names of feature annotations in an annotation + annotationKeys := make([]string, 0, len(featureAnnotations)) + for key := range featureAnnotations { + // Drop the ns part for annotations in the default ns + annotationKeys = append(annotationKeys, strings.TrimPrefix(key, nfdv1alpha1.FeatureAnnotationNs+"/")) + } + sort.Strings(annotationKeys) + nfdAnnotations[m.instanceAnnotation(nfdv1alpha1.FeatureAnnotationsTrackingAnnotation)] = strings.Join(annotationKeys, ",") + for k, v := range featureAnnotations { + annotations[k] = v + } + } + + if len(nfdAnnotations) > 0 { + for k, v := range nfdAnnotations { + annotations[k] = v + } } // Create JSON patches for changes in labels and annotations oldLabels := stringToNsNames(node.Annotations[m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation)], nfdv1alpha1.FeatureLabelNs) + oldAnnotations := stringToNsNames(node.Annotations[m.instanceAnnotation(nfdv1alpha1.FeatureAnnotationsTrackingAnnotation)], nfdv1alpha1.FeatureAnnotationNs) patches := createPatches(oldLabels, node.Labels, labels, "/metadata/labels") - patches = append(patches, - createPatches( - []string{ - m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation), - m.instanceAnnotation(nfdv1alpha1.ExtendedResourceAnnotation), - // Clean up deprecated/stale nfd version annotations - m.instanceAnnotation(nfdv1alpha1.MasterVersionAnnotation), - m.instanceAnnotation(nfdv1alpha1.WorkerVersionAnnotation)}, - node.Annotations, - annotations, - "/metadata/annotations")...) + oldAnnotations = append(oldAnnotations, []string{ + m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation), + m.instanceAnnotation(nfdv1alpha1.ExtendedResourceAnnotation), + m.instanceAnnotation(nfdv1alpha1.FeatureAnnotationsTrackingAnnotation), + // Clean up deprecated/stale nfd version annotations + m.instanceAnnotation(nfdv1alpha1.MasterVersionAnnotation), + m.instanceAnnotation(nfdv1alpha1.WorkerVersionAnnotation)}...) + patches = append(patches, createPatches(oldAnnotations, node.Annotations, annotations, "/metadata/annotations")...) // patch node status with extended resource changes statusPatches := m.createExtendedResourcePatches(node, extendedResources) @@ -1379,3 +1407,27 @@ func (m *nfdMaster) nfdAPIUpdateHandlerWithLeaderElection() { leaderElector.Run(ctx) } + +// Filter annotations by namespace. i.e. adds the possibly missing default namespace for annotations +func (m *nfdMaster) filterFeatureAnnotations(annotations map[string]string) map[string]string { + outAnnotations := make(map[string]string) + + for annotation, value := range annotations { + // Add possibly missing default ns + annotation := addNs(annotation, nfdv1alpha1.FeatureAnnotationNs) + + ns, _ := splitNs(annotation) + + // Check annotation namespace, filter out if ns is not whitelisted + if ns != nfdv1alpha1.FeatureAnnotationNs && !strings.HasSuffix(ns, nfdv1alpha1.FeatureAnnotationSubNsSuffix) { + // If the namespace is denied, and not present in the extraLabelNs, label will be ignored + if ns == "kubernetes.io" || strings.HasSuffix(ns, ".kubernetes.io") || ns == nfdv1alpha1.AnnotationNs { + klog.ErrorS(fmt.Errorf("namespace %v is not allowed", ns), fmt.Sprintf("Ignoring annotation %v\n", annotation)) + continue + } + } + + outAnnotations[annotation] = value + } + return outAnnotations +} diff --git a/test/e2e/data/nodefeaturerule-5.yaml b/test/e2e/data/nodefeaturerule-5.yaml new file mode 100644 index 000000000..6cdccf537 --- /dev/null +++ b/test/e2e/data/nodefeaturerule-5.yaml @@ -0,0 +1,19 @@ +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: e2e-feature-annotations-test +spec: + rules: + # Positive test expected to set the annotations + - name: "e2e-annotation-test" + annotations: + defaul-ns-annotation: "foo" + feature.node.kubernetes.io/defaul-ns-annotation-2: "bar" + custom.vendor.io/feature: "baz" + kubernetes.io/feature: "denied" + subns.kubernetes.io/blah: "denied" + nfd.node.kubernetes.io/xyz: "denied" + matchFeatures: + - feature: "fake.flag" + matchExpressions: + "flag_1": {op: Exists} diff --git a/test/e2e/node_feature_discovery_test.go b/test/e2e/node_feature_discovery_test.go index ef9ca1ae7..b7bef9938 100644 --- a/test/e2e/node_feature_discovery_test.go +++ b/test/e2e/node_feature_discovery_test.go @@ -803,6 +803,25 @@ core: eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchCapacity(expectedCapacity, nodes)) eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes)) + By("Creating NodeFeatureRules #5") + Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-5.yaml")).NotTo(HaveOccurred()) + + By("Verifying node annotations from NodeFeatureRules #5") + expectedAnnotations["*"] = k8sAnnotations{ + nfdv1alpha1.FeatureLabelNs + "/defaul-ns-annotation": "foo", + nfdv1alpha1.FeatureLabelNs + "/defaul-ns-annotation-2": "bar", + "custom.vendor.example/feature": "baz", + } + eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes)) + + By("Deleting NodeFeatureRule object") + err = nfdClient.NfdV1alpha1().NodeFeatureRules().Delete(ctx, " e2e-feature-annotations-test", metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying node annotations from NodeFeatureRules #5 are deleted") + expectedAnnotations["*"] = k8sAnnotations{} + eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes)) + By("Deleting nfd-worker daemonset") err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(ctx, workerDS.Name, metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred())