mirrors/node-feature-discovery
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/node-feature-discovery.git synced 2024-12-14 11:57:51 +00:00
Code Activity

Discover node features as annotations

Browse source 
Signed-off-by: Carlos Eduardo Arango Gutierrez <eduardoa@nvidia.com>
Co-authored-by: bebc <mchf1990212@gmail.com>
Co-authored-by: Markus Lehtonen <markus.lehtonen@intel.com>
This commit is contained in:
Carlos Eduardo Arango Gutierrez 2023-10-13 17:36:32 +02:00
parent 9865b4a880
commit c0063be4f4
No known key found for this signature in database
GPG key ID: 42D9CB42F300A852
12 changed files with 207 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
deployment/base/nfd-crds/nfd-api-crds.yaml
View file

@ -153,6 +153,11 @@ spec:
description: Rule defines a rule for node customization such as
labeling.
properties:
annotations:
additionalProperties:
type: string
description: Annotations to create if the rule matches.
type: object
extendedResources:
additionalProperties:
type: string

5
deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml
View file

@ -153,6 +153,11 @@ spec:
description: Rule defines a rule for node customization such as
labeling.
properties:
annotations:
additionalProperties:
type: string
description: Annotations to create if the rule matches.
type: object
extendedResources:
additionalProperties:
type: string

4
docs/get-started/introduction.md
View file

@ -18,8 +18,8 @@ sort: 1
This software enables node feature discovery for Kubernetes. It detects
hardware features available on each node in a Kubernetes cluster, and
advertises those features using node labels and optionally node extended
resources and node taints. Node Feature Discovery is compatible with any recent
version of Kubernetes (v1.21+).
resources, annotations and node taints. Node Feature Discovery is compatible
with any recent version of Kubernetes (v1.21+).
NFD consists of four software components:

48
docs/usage/customization-guide.md
View file

@ -594,6 +594,54 @@ details.
> labels specified in the `labels` field will override anything
> originating from `labelsTemplate`.
#### Node Annotations
The `.annotations` field is a list of features to be advertised as annotations.
Take this rule as a referential example:
```yaml
apiVersion: nfd.k8s-sigs.io/v1alpha1
kind: NodeFeatureRule
metadata:
name: feature-annotations-example
spec:
rules:
- name: "annotation-example"
annotations:
defaul-ns-annotation: "foo"
feature.node.kubernetes.io/defaul-ns-annotation-2: "bar"
custom.vendor.io/feature: "baz"
matchFeatures:
- feature: kernel.version
matchExpressions:
major: {op: Exists}
```
This will yield into the following node annotations:
```yaml
annotations:
...
feature.node.kubernetes.io/defaul-ns-annotation: "foo"
feature.node.kubernetes.io/defaul-ns-annotation-2: "bar"
custom.vendor.io/feature: "baz"
...
```
NFD enforces some limitations to the namespace (or prefix)/ of the annotations:
- `kubernetes.io/` and its sub-namespaces (like `sub.ns.kubernetes.io/`) cannot
generally be used
- the only exception is `feature.node.kubernetes.io/` and its sub-namespaces
(like `sub.ns.feature.node.kubernetes.io`)
- unprefixed names will get prefixed with `feature.node.kubernetes.io/`
automatically (e.g. `foo` becomes `feature.node.kubernetes.io/foo`)
> **NOTE:** The `annotations` field has will only advertise features via node
> annotations the features won't be advertised as node labels unless they are
> specified in the `labels` field.
#### Taints
*taints* is a list of taint entries and each entry can have `key`, `value` and `effect`,

9
pkg/apis/nfd/v1alpha1/annotations_labels.go
View file

@ -60,9 +60,18 @@ const (
// NodeTaintsAnnotation is the annotation that holds the taints that nfd-master set on the node
NodeTaintsAnnotation = AnnotationNs + "/taints"
// FeatureAnnotationsTrackingAnnotation is the annotation that holds all feature annotations that nfd-master set on the node
FeatureAnnotationsTrackingAnnotation = AnnotationNs + "/feature-annotations"
// NodeFeatureObjNodeNameLabel is the label that specifies which node the
// NodeFeature object is targeting. Creators of NodeFeature objects must
// set this label and consumers of the objects are supposed to use the
// label for filtering features designated for a certain node.
NodeFeatureObjNodeNameLabel = "nfd.node.kubernetes.io/node-name"
// FeatureAnnotationNs is the (default) namespace for feature annotations.
FeatureAnnotationNs = "feature.node.kubernetes.io"
// FeatureAnnotationSubNsSuffix is the suffix for allowed feature annotation sub-namespaces.
FeatureAnnotationSubNsSuffix = "." + FeatureAnnotationNs
)

4
pkg/apis/nfd/v1alpha1/rule.go
View file

@ -24,6 +24,7 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/klog/v2"
"sigs.k8s.io/node-feature-discovery/pkg/utils"
)
@ -32,6 +33,7 @@ import (
type RuleOutput struct {
ExtendedResources map[string]string
Labels map[string]string
Annotations map[string]string
Vars map[string]string
Taints []corev1.Taint
}
@ -101,7 +103,7 @@ func (r *Rule) Execute(features *Features) (RuleOutput, error) {
vars[k] = v
}
ret := RuleOutput{ExtendedResources: extendedResources, Labels: labels, Vars: vars, Taints: r.Taints}
ret := RuleOutput{ExtendedResources: extendedResources, Labels: labels, Vars: vars, Taints: r.Taints, Annotations: r.Annotations}
klog.V(2).InfoS("rule matched", "ruleName", r.Name, "ruleOutput", utils.DelayedDumper(ret))
return ret, nil
}

4
pkg/apis/nfd/v1alpha1/types.go
View file

@ -146,6 +146,10 @@ type Rule struct {
// +optional
LabelsTemplate string `json:"labelsTemplate"`
// Annotations to create if the rule matches.
// +optional
Annotations map[string]string `json:"annotations"`
// Vars is the variables to store if the rule matches. Variables do not
// directly inflict any changes in the node object. However, they can be
// referenced from other rules enabling more complex rule hierarchies,

7
pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go
View file

@ -513,6 +513,13 @@ func (in *Rule) DeepCopyInto(out *Rule) {
(*out)[key] = val
}
}
if in.Annotations != nil {
in, out := &in.Annotations, &out.Annotations
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.Vars != nil {
in, out := &in.Vars, &out.Vars
*out = make(map[string]string, len(*in))

17
pkg/nfd-master/nfd-master-internal_test.go
View file

@ -138,6 +138,12 @@ func TestUpdateNodeObject(t *testing.T) {
}
sort.Strings(fakeFeatureLabelNames)
fakeFeatureAnnotationsNames := make([]string, 0, len(fakeFeatureLabels))
for k := range fakeAnnotations {
fakeFeatureAnnotationsNames = append(fakeFeatureAnnotationsNames, strings.TrimPrefix(k, nfdv1alpha1.FeatureAnnotationNs+"/"))
}
sort.Strings(fakeFeatureAnnotationsNames)
fakeExtResourceNames := make([]string, 0, len(fakeExtResources))
for k := range fakeExtResources {
fakeExtResourceNames = append(fakeExtResourceNames, strings.TrimPrefix(k, nfdv1alpha1.FeatureLabelNs+"/"))
@ -162,6 +168,7 @@ func TestUpdateNodeObject(t *testing.T) {
// Create a list of expected node metadata patches
metadataPatches := []apihelper.JsonPatch{
apihelper.NewJsonPatch("replace", "/metadata/annotations", nfdv1alpha1.AnnotationNs+"/feature-labels", strings.Join(fakeFeatureLabelNames, ",")),
apihelper.NewJsonPatch("add", "/metadata/annotations", nfdv1alpha1.FeatureAnnotationsTrackingAnnotation, strings.Join(fakeFeatureAnnotationsNames, ",")),
apihelper.NewJsonPatch("add", "/metadata/annotations", nfdv1alpha1.AnnotationNs+"/extended-resources", strings.Join(fakeExtResourceNames, ",")),
apihelper.NewJsonPatch("remove", "/metadata/labels", nfdv1alpha1.FeatureLabelNs+"/old-feature", ""),
}
@ -176,7 +183,7 @@ func TestUpdateNodeObject(t *testing.T) {
mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(mockNode, nil).Twice()
mockAPIHelper.On("PatchNodeStatus", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(statusPatches))).Return(nil)
mockAPIHelper.On("PatchNode", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(metadataPatches))).Return(nil)
err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
Convey("Error is nil", func() {
So(err, ShouldBeNil)
@ -186,7 +193,7 @@ func TestUpdateNodeObject(t *testing.T) {
Convey("When I fail to update the node with feature labels", func() {
expectedError := fmt.Errorf("no client is passed, client: <nil>")
mockAPIHelper.On("GetClient").Return(nil, expectedError)
err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
Convey("Error is produced", func() {
So(err, ShouldResemble, expectedError)
@ -196,7 +203,7 @@ func TestUpdateNodeObject(t *testing.T) {
Convey("When I fail to get a mock client while updating feature labels", func() {
expectedError := fmt.Errorf("no client is passed, client: <nil>")
mockAPIHelper.On("GetClient").Return(nil, expectedError)
err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
Convey("Error is produced", func() {
So(err, ShouldResemble, expectedError)
@ -207,7 +214,7 @@ func TestUpdateNodeObject(t *testing.T) {
expectedError := errors.New("fake error")
mockAPIHelper.On("GetClient").Return(mockClient, nil)
mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(nil, expectedError).Twice()
err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
Convey("Error is produced", func() {
So(err, ShouldEqual, expectedError)
@ -220,7 +227,7 @@ func TestUpdateNodeObject(t *testing.T) {
mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(mockNode, nil).Twice()
mockAPIHelper.On("PatchNodeStatus", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(statusPatches))).Return(nil)
mockAPIHelper.On("PatchNode", mockClient, mockNodeName, mock.Anything).Return(expectedError).Twice()
err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
Convey("Error is produced", func() {
So(err.Error(), ShouldEndWith, expectedError.Error())

96
pkg/nfd-master/nfd-master.go
View file

@ -465,7 +465,7 @@ func (m *nfdMaster) prune() error {
klog.InfoS("pruning node...", "nodeName", node.Name)
// Prune labels and extended resources
err := m.updateNodeObject(cli, node.Name, Labels{}, Annotations{}, ExtendedResources{}, []corev1.Taint{})
err := m.updateNodeObject(cli, node.Name, Labels{}, Annotations{}, Annotations{}, ExtendedResources{}, []corev1.Taint{})
if err != nil {
nodeUpdateFailures.Inc()
return fmt.Errorf("failed to prune node %q: %v", node.Name, err)
@ -837,12 +837,12 @@ func filterExtendedResource(name, value string, features *nfdv1alpha1.Features)
return q.String(), nil
}
func (m *nfdMaster) refreshNodeFeatures(cli *kubernetes.Clientset, nodeName string, annotations Annotations, labels map[string]string, features *nfdv1alpha1.Features) error {
func (m *nfdMaster) refreshNodeFeatures(cli *kubernetes.Clientset, nodeName string, nfdAnnotations Annotations, labels map[string]string, features *nfdv1alpha1.Features) error {
if labels == nil {
labels = make(map[string]string)
}
crLabels, crExtendedResources, crTaints := m.processNodeFeatureRule(nodeName, features)
crLabels, crAnnotations, crExtendedResources, crTaints := m.processNodeFeatureRule(nodeName, features)
// Mix in CR-originated labels
for k, v := range crLabels {
@ -859,12 +859,16 @@ func (m *nfdMaster) refreshNodeFeatures(cli *kubernetes.Clientset, nodeName stri
}
extendedResources = filterExtendedResources(features, extendedResources)
// Annotations
featureAnnotations := m.filterFeatureAnnotations(crAnnotations)
// Taints
var taints []corev1.Taint
if m.config.EnableTaints {
taints = filterTaints(crTaints)
}
err := m.updateNodeObject(cli, nodeName, labels, annotations, extendedResources, taints)
err := m.updateNodeObject(cli, nodeName, labels, nfdAnnotations, featureAnnotations, extendedResources, taints)
if err != nil {
klog.ErrorS(err, "failed to update node", "nodeName", nodeName)
return err
@ -974,13 +978,14 @@ func authorizeClient(c context.Context, checkNodeName bool, nodeName string) err
return nil
}
func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha1.Features) (Labels, ExtendedResources, []corev1.Taint) {
func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha1.Features) (Labels, Annotations, ExtendedResources, []corev1.Taint) {
if m.nfdController == nil {
return nil, nil, nil
return nil, nil, nil, nil
}
extendedResources := ExtendedResources{}
labels := make(map[string]string)
annotations := make(map[string]string)
var taints []corev1.Taint
ruleSpecs, err := m.nfdController.ruleLister.List(k8sLabels.Everything())
sort.Slice(ruleSpecs, func(i, j int) bool {
@ -989,7 +994,7 @@ func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha
if err != nil {
klog.ErrorS(err, "failed to list NodeFeatureRule resources")
return nil, nil, nil
return nil, nil, nil, nil
}
// Process all rule CRs
@ -1016,6 +1021,9 @@ func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha
for k, v := range ruleOut.ExtendedResources {
extendedResources[k] = v
}
for k, v := range ruleOut.Annotations {
annotations[k] = v
}
// Feed back rule output to features map for subsequent rules to match
features.InsertAttributeFeatures(nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Labels)
@ -1026,13 +1034,13 @@ func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha
processingTime := time.Since(processStart)
klog.V(2).InfoS("processed NodeFeatureRule objects", "nodeName", nodeName, "objectCount", len(ruleSpecs), "duration", processingTime)
return labels, extendedResources, taints
return labels, annotations, extendedResources, taints
}
// updateNodeObject ensures the Kubernetes node object is up to date,
// creating new labels and extended resources where necessary and removing
// outdated ones. Also updates the corresponding annotations.
func (m *nfdMaster) updateNodeObject(cli *kubernetes.Clientset, nodeName string, labels Labels, annotations Annotations, extendedResources ExtendedResources, taints []corev1.Taint) error {
func (m *nfdMaster) updateNodeObject(cli *kubernetes.Clientset, nodeName string, labels Labels, nfdAnnotations, featureAnnotations Annotations, extendedResources ExtendedResources, taints []corev1.Taint) error {
if cli == nil {
return fmt.Errorf("no client is passed, client: %v", cli)
}
@ -1051,7 +1059,7 @@ func (m *nfdMaster) updateNodeObject(cli *kubernetes.Clientset, nodeName string,
labelKeys = append(labelKeys, strings.TrimPrefix(key, nfdv1alpha1.FeatureLabelNs+"/"))
}
sort.Strings(labelKeys)
annotations[m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation)] = strings.Join(labelKeys, ",")
nfdAnnotations[m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation)] = strings.Join(labelKeys, ",")
}
// Store names of extended resources in an annotation
@ -1062,23 +1070,43 @@ func (m *nfdMaster) updateNodeObject(cli *kubernetes.Clientset, nodeName string,
extendedResourceKeys = append(extendedResourceKeys, strings.TrimPrefix(key, nfdv1alpha1.FeatureLabelNs+"/"))
}
sort.Strings(extendedResourceKeys)
annotations[m.instanceAnnotation(nfdv1alpha1.ExtendedResourceAnnotation)] = strings.Join(extendedResourceKeys, ",")
nfdAnnotations[m.instanceAnnotation(nfdv1alpha1.ExtendedResourceAnnotation)] = strings.Join(extendedResourceKeys, ",")
}
// Store feature annotations
annotations := make(Annotations)
if len(featureAnnotations) > 0 {
// Store names of feature annotations in an annotation
annotationKeys := make([]string, 0, len(featureAnnotations))
for key := range featureAnnotations {
// Drop the ns part for annotations in the default ns
annotationKeys = append(annotationKeys, strings.TrimPrefix(key, nfdv1alpha1.FeatureAnnotationNs+"/"))
}
sort.Strings(annotationKeys)
nfdAnnotations[m.instanceAnnotation(nfdv1alpha1.FeatureAnnotationsTrackingAnnotation)] = strings.Join(annotationKeys, ",")
for k, v := range featureAnnotations {
annotations[k] = v
}
}
if len(nfdAnnotations) > 0 {
for k, v := range nfdAnnotations {
annotations[k] = v
}
}
// Create JSON patches for changes in labels and annotations
oldLabels := stringToNsNames(node.Annotations[m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation)], nfdv1alpha1.FeatureLabelNs)
oldAnnotations := stringToNsNames(node.Annotations[m.instanceAnnotation(nfdv1alpha1.FeatureAnnotationsTrackingAnnotation)], nfdv1alpha1.FeatureAnnotationNs)
patches := createPatches(oldLabels, node.Labels, labels, "/metadata/labels")
patches = append(patches,
createPatches(
[]string{
m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation),
m.instanceAnnotation(nfdv1alpha1.ExtendedResourceAnnotation),
// Clean up deprecated/stale nfd version annotations
m.instanceAnnotation(nfdv1alpha1.MasterVersionAnnotation),
m.instanceAnnotation(nfdv1alpha1.WorkerVersionAnnotation)},
node.Annotations,
annotations,
"/metadata/annotations")...)
oldAnnotations = append(oldAnnotations, []string{
m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation),
m.instanceAnnotation(nfdv1alpha1.ExtendedResourceAnnotation),
m.instanceAnnotation(nfdv1alpha1.FeatureAnnotationsTrackingAnnotation),
// Clean up deprecated/stale nfd version annotations
m.instanceAnnotation(nfdv1alpha1.MasterVersionAnnotation),
m.instanceAnnotation(nfdv1alpha1.WorkerVersionAnnotation)}...)
patches = append(patches, createPatches(oldAnnotations, node.Annotations, annotations, "/metadata/annotations")...)
// patch node status with extended resource changes
statusPatches := m.createExtendedResourcePatches(node, extendedResources)
@ -1379,3 +1407,27 @@ func (m *nfdMaster) nfdAPIUpdateHandlerWithLeaderElection() {
leaderElector.Run(ctx)
}
// Filter annotations by namespace. i.e. adds the possibly missing default namespace for annotations
func (m *nfdMaster) filterFeatureAnnotations(annotations map[string]string) map[string]string {
outAnnotations := make(map[string]string)
for annotation, value := range annotations {
// Add possibly missing default ns
annotation := addNs(annotation, nfdv1alpha1.FeatureAnnotationNs)
ns, _ := splitNs(annotation)
// Check annotation namespace, filter out if ns is not whitelisted
if ns != nfdv1alpha1.FeatureAnnotationNs && !strings.HasSuffix(ns, nfdv1alpha1.FeatureAnnotationSubNsSuffix) {
// If the namespace is denied, and not present in the extraLabelNs, label will be ignored
if ns == "kubernetes.io" || strings.HasSuffix(ns, ".kubernetes.io") || ns == nfdv1alpha1.AnnotationNs {
klog.ErrorS(fmt.Errorf("namespace %v is not allowed", ns), fmt.Sprintf("Ignoring annotation %v\n", annotation))
continue
}
}
outAnnotations[annotation] = value
}
return outAnnotations
}

19
test/e2e/data/nodefeaturerule-5.yaml Normal file
View file

@ -0,0 +1,19 @@
apiVersion: nfd.k8s-sigs.io/v1alpha1
kind: NodeFeatureRule
metadata:
name: e2e-feature-annotations-test
spec:
rules:
# Positive test expected to set the annotations
- name: "e2e-annotation-test"
annotations:
defaul-ns-annotation: "foo"
feature.node.kubernetes.io/defaul-ns-annotation-2: "bar"
custom.vendor.io/feature: "baz"
kubernetes.io/feature: "denied"
subns.kubernetes.io/blah: "denied"
nfd.node.kubernetes.io/xyz: "denied"
matchFeatures:
- feature: "fake.flag"
matchExpressions:
"flag_1": {op: Exists}

19
test/e2e/node_feature_discovery_test.go
View file

@ -803,6 +803,25 @@ core:
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchCapacity(expectedCapacity, nodes))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))
By("Creating NodeFeatureRules #5")
Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-5.yaml")).NotTo(HaveOccurred())
By("Verifying node annotations from NodeFeatureRules #5")
expectedAnnotations["*"] = k8sAnnotations{
nfdv1alpha1.FeatureLabelNs + "/defaul-ns-annotation": "foo",
nfdv1alpha1.FeatureLabelNs + "/defaul-ns-annotation-2": "bar",
"custom.vendor.example/feature": "baz",
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))
By("Deleting NodeFeatureRule object")
err = nfdClient.NfdV1alpha1().NodeFeatureRules().Delete(ctx, " e2e-feature-annotations-test", metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
By("Verifying node annotations from NodeFeatureRules #5 are deleted")
expectedAnnotations["*"] = k8sAnnotations{}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))
By("Deleting nfd-worker daemonset")
err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(ctx, workerDS.Name, metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())