mirrors/node-feature-discovery
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/node-feature-discovery.git synced 2024-12-14 11:57:51 +00:00
Code Activity

feat/nfd-master: support CR restrictions

Browse source 
Signed-off-by: AhmedGrati <ahmedgrati1999@gmail.com>
This commit is contained in:
AhmedGrati 2024-02-15 15:57:50 +01:00
parent c1db298de8
commit 7bad0d583c
6 changed files with 453 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -2,6 +2,8 @@ module sigs.k8s.io/node-feature-discovery
go 1.22.2
toolchain go1.22.0
require (
github.com/fsnotify/fsnotify v1.7.0
github.com/golang/protobuf v1.5.4

56
pkg/nfd-master/nfd-api-controller.go
View file

@ -17,6 +17,7 @@ limitations under the License.
package nfdmaster
import (
"context"
"fmt"
"time"
@ -26,7 +27,6 @@ import (
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
nfdclientset "sigs.k8s.io/node-feature-discovery/api/generated/clientset/versioned"
nfdscheme "sigs.k8s.io/node-feature-discovery/api/generated/clientset/versioned/scheme"
nfdinformers "sigs.k8s.io/node-feature-discovery/api/generated/informers/externalversions"
nfdinformersv1alpha1 "sigs.k8s.io/node-feature-discovery/api/generated/informers/externalversions/nfd/v1alpha1"
@ -65,6 +65,8 @@ func newNfdController(config *restclient.Config, nfdApiControllerOptions nfdApiC
updateOneNodeChan: make(chan string),
updateAllNodeFeatureGroupsChan: make(chan struct{}, 1),
updateNodeFeatureGroupChan: make(chan string),
k8sClient: nfdApiControllerOptions.K8sClient,
nodeFeatureNamespaceSelector: nfdApiControllerOptions.NodeFeatureNamespaceSelector,
}
nfdClient := nfdclientset.NewForConfigOrDie(config)
@ -89,25 +91,28 @@ func newNfdController(config *restclient.Config, nfdApiControllerOptions nfdApiC
AddFunc: func(obj interface{}) {
nfr := obj.(*nfdv1alpha1.NodeFeature)
klog.V(2).InfoS("NodeFeature added", "nodefeature", klog.KObj(nfr))
c.updateOneNode("NodeFeature", nfr)
if !nfdApiControllerOptions.DisableNodeFeatureGroup {
c.updateAllNodeFeatureGroups()
if c.isNamespaceSelected(nfr.Namespace) {
c.updateOneNode("NodeFeature", nfr)
} else {
klog.InfoS("NodeFeature not in selected namespace", "namespace", nfr.Namespace, "name", nfr.Name)
}
},
UpdateFunc: func(oldObj, newObj interface{}) {
nfr := newObj.(*nfdv1alpha1.NodeFeature)
klog.V(2).InfoS("NodeFeature updated", "nodefeature", klog.KObj(nfr))
c.updateOneNode("NodeFeature", nfr)
if !nfdApiControllerOptions.DisableNodeFeatureGroup {
c.updateAllNodeFeatureGroups()
if c.isNamespaceSelected(nfr.Namespace) {
c.updateOneNode("NodeFeature", nfr)
} else {
klog.InfoS("NodeFeature not in selected namespace", "namespace", nfr.Namespace, "name", nfr.Name)
}
},
DeleteFunc: func(obj interface{}) {
nfr := obj.(*nfdv1alpha1.NodeFeature)
klog.V(2).InfoS("NodeFeature deleted", "nodefeature", klog.KObj(nfr))
c.updateOneNode("NodeFeature", nfr)
if !nfdApiControllerOptions.DisableNodeFeatureGroup {
c.updateAllNodeFeatureGroups()
if c.isNamespaceSelected(nfr.Namespace) {
c.updateOneNode("NodeFeature", nfr)
} else {
klog.InfoS("NodeFeature not in selected namespace", "namespace", nfr.Namespace, "name", nfr.Name)
}
},
}); err != nil {
@ -209,6 +214,37 @@ func (c *nfdController) updateOneNode(typ string, obj metav1.Object) {
c.updateOneNodeChan <- nodeName
}
func (c *nfdController) isNamespaceSelected(namespace string) bool {
// no namespace restrictions are made here
if c.nodeFeatureNamespaceSelector == nil {
return true
}
labelMap, err := metav1.LabelSelectorAsSelector(c.nodeFeatureNamespaceSelector)
if err != nil {
klog.ErrorS(err, "failed to convert label selector to map", "selector", c.nodeFeatureNamespaceSelector)
return false
}
listOptions := metav1.ListOptions{
LabelSelector: labelMap.String(),
}
namespaces, err := c.k8sClient.CoreV1().Namespaces().List(context.Background(), listOptions)
if err != nil {
klog.ErrorS(err, "failed to query namespaces", "listOptions", listOptions)
return false
}
for _, ns := range namespaces.Items {
if ns.Name == namespace {
return true
}
}
return false
}
func (c *nfdController) updateAllNodes() {
select {
case c.updateAllNodesChan <- struct{}{}:

56
pkg/nfd-master/nfd-api-controller_test.go
View file

@ -23,6 +23,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/api/nfd/v1alpha1"
fakeclient "k8s.io/client-go/kubernetes/fake"
corev1 "k8s.io/api/core/v1"
)
func TestGetNodeNameForObj(t *testing.T) {
@ -42,3 +44,57 @@ func TestGetNodeNameForObj(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, n, "node-1")
}
func newTestNamespace(name string) *corev1.Namespace{
return &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: map[string]string{
"name": name,
},
},
}
}
func TestIsNamespaceAllowed(t *testing.T) {
fakeCli := fakeclient.NewSimpleClientset(newTestNamespace("fake"))
c := &nfdController{
k8sClient: fakeCli,
}
testcases := []struct {
name string
objectNamespace string
nodeFeatureNamespaceSelector *metav1.LabelSelector
expectedResult bool
}{
{
name: "namespace not allowed",
objectNamespace: "random",
nodeFeatureNamespaceSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "name",
Operator: metav1.LabelSelectorOpIn,
Values: []string{"fake"},
},
},
},
expectedResult: false,
},
{
name: "namespace is allowed",
objectNamespace: "fake",
nodeFeatureNamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"name": "fake"},
},
expectedResult: false,
},
}
for _, tc := range testcases {
c.nodeFeatureNamespaceSelector = tc.nodeFeatureNamespaceSelector
res := c.isNamespaceSelected(tc.name)
assert.Equal(t, res, tc.expectedResult)
}
}

84
pkg/nfd-master/nfd-master-internal_test.go
View file

@ -509,15 +509,16 @@ func TestFilterLabels(t *testing.T) {
func TestCreatePatches(t *testing.T) {
Convey("When creating JSON patches", t, func() {
existingItems := map[string]string{"key-1": "val-1", "key-2": "val-2", "key-3": "val-3"}
overwriteKeys := true
jsonPath := "/root"
Convey("When when there are neither itmes to remoe nor to add or update", func() {
p := createPatches([]string{"foo", "bar"}, existingItems, map[string]string{}, jsonPath)
Convey("When there are neither itmes to remoe nor to add or update", func() {
p := createPatches([]string{"foo", "bar"}, existingItems, map[string]string{}, jsonPath, overwriteKeys)
So(len(p), ShouldEqual, 0)
})
Convey("When when there are itmes to remoe but none to add or update", func() {
p := createPatches([]string{"key-2", "key-3", "foo"}, existingItems, map[string]string{}, jsonPath)
Convey("When there are itmes to remoe but none to add or update", func() {
p := createPatches([]string{"key-2", "key-3", "foo"}, existingItems, map[string]string{}, jsonPath, overwriteKeys)
expected := []utils.JsonPatch{
utils.NewJsonPatch("remove", jsonPath, "key-2", ""),
utils.NewJsonPatch("remove", jsonPath, "key-3", ""),
@ -525,9 +526,9 @@ func TestCreatePatches(t *testing.T) {
So(sortJsonPatches(p), ShouldResemble, sortJsonPatches(expected))
})
Convey("When when there are no itmes to remove but new items to add", func() {
Convey("When there are no itmes to remove but new items to add", func() {
newItems := map[string]string{"new-key": "new-val", "key-1": "new-1"}
p := createPatches([]string{"key-1"}, existingItems, newItems, jsonPath)
p := createPatches([]string{"key-1"}, existingItems, newItems, jsonPath, overwriteKeys)
expected := []utils.JsonPatch{
utils.NewJsonPatch("add", jsonPath, "new-key", newItems["new-key"]),
utils.NewJsonPatch("replace", jsonPath, "key-1", newItems["key-1"]),
@ -535,9 +536,9 @@ func TestCreatePatches(t *testing.T) {
So(sortJsonPatches(p), ShouldResemble, sortJsonPatches(expected))
})
Convey("When when there are items to remove add and update", func() {
Convey("When there are items to remove add and update", func() {
newItems := map[string]string{"new-key": "new-val", "key-2": "new-2", "key-4": "val-4"}
p := createPatches([]string{"key-1", "key-2", "key-3", "foo"}, existingItems, newItems, jsonPath)
p := createPatches([]string{"key-1", "key-2", "key-3", "foo"}, existingItems, newItems, jsonPath, overwriteKeys)
expected := []utils.JsonPatch{
utils.NewJsonPatch("add", jsonPath, "new-key", newItems["new-key"]),
utils.NewJsonPatch("add", jsonPath, "key-4", newItems["key-4"]),
@ -547,6 +548,16 @@ func TestCreatePatches(t *testing.T) {
}
So(sortJsonPatches(p), ShouldResemble, sortJsonPatches(expected))
})
Convey("When overwrite of keys is denied and there is already an existant key", func() {
overwriteKeys = false
newItems := map[string]string{"key-1": "new-2", "key-4": "val-4"}
p := createPatches([]string{}, existingItems, newItems, jsonPath, overwriteKeys)
expected := []utils.JsonPatch{
utils.NewJsonPatch("add", jsonPath, "key-4", newItems["key-4"]),
}
So(sortJsonPatches(p), ShouldResemble, sortJsonPatches(expected))
})
})
}
@ -896,3 +907,60 @@ func TestGetDynamicValue(t *testing.T) {
})
}
}
func TestFilterTaints(t *testing.T) {
testcases := []struct {
name string
taints []corev1.Taint
maxTaints int
expectedResult []corev1.Taint
}{
{
name: "no restriction on the number of taints",
taints: []corev1.Taint{
{
Key: "feature.node.kubernetes.io/key1",
Value: "dummy",
Effect: corev1.TaintEffectNoSchedule,
},
},
maxTaints: 0,
expectedResult: []corev1.Taint{
{
Key: "feature.node.kubernetes.io/key1",
Value: "dummy",
Effect: corev1.TaintEffectNoSchedule,
},
},
},
{
name: "max of 1 Taint should be generated",
taints: []corev1.Taint{
{
Key: "feature.node.kubernetes.io/key1",
Value: "dummy",
Effect: corev1.TaintEffectNoSchedule,
},
{
Key: "feature.node.kubernetes.io/key2",
Value: "dummy",
Effect: corev1.TaintEffectNoSchedule,
},
},
maxTaints: 1,
expectedResult: []corev1.Taint{},
},
}
mockMaster := newFakeMaster(nil)
for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
mockMaster.config.Restrictions.MaxTaintsPerCR = tc.maxTaints
res := mockMaster.filterTaints(tc.taints)
Convey("The expected number of taints should be correct", t, func() {
So(len(res), ShouldEqual, len(tc.expectedResult))
})
})
}
}

62
pkg/nfd-master/nfd-master.go
View file

@ -75,6 +75,16 @@ type ExtendedResources map[string]string
// Annotations are used for NFD-related node metadata
type Annotations map[string]string
// Restrictions contains the restrictions on the NF and NFR Crs
type Restrictions struct {
AllowedNamespaces *metav1.LabelSelector
MaxLabelsPerCR int
MaxTaintsPerCR int
MaxExtendedResourcesPerCR int
DenyNodeFeatureLabels bool
OverwriteLabels bool
}
// NFDConfig contains the configuration settings of NfdMaster.
type NFDConfig struct {
AutoDefaultNs bool
@ -88,6 +98,7 @@ type NFDConfig struct {
LeaderElection LeaderElectionConfig
NfdApiParallelism int
Klog klogutils.KlogConfigOpts
Restrictions Restrictions
}
// LeaderElectionConfig contains the configuration for leader election
@ -273,6 +284,13 @@ func newDefaultConfig() *NFDConfig {
RenewDeadline: utils.DurationVal{Duration: time.Duration(10) * time.Second},
},
Klog: make(map[string]string),
Restrictions: Restrictions{
MaxLabelsPerCR: 0,
MaxTaintsPerCR: 0,
MaxExtendedResourcesPerCR: 0,
OverwriteLabels: true,
DenyNodeFeatureLabels: false,
},
}
}
@ -620,7 +638,7 @@ func (m *nfdMaster) updateMasterNode() error {
p := createPatches([]string{m.instanceAnnotation(nfdv1alpha1.MasterVersionAnnotation)},
node.Annotations,
nil,
"/metadata/annotations")
"/metadata/annotations", true)
err = patchNode(m.k8sClient, node.Name, p)
if err != nil {
@ -661,6 +679,11 @@ func (m *nfdMaster) filterFeatureLabels(labels Labels, features *nfdv1alpha1.Fea
}
}
if m.config.Restrictions.MaxLabelsPerCR > 0 && len(outLabels) > m.config.Restrictions.MaxLabelsPerCR {
klog.InfoS("number of labels in the request exceeds the restriction maximum labels per CR, skipping")
outLabels = Labels{}
}
return outLabels, extendedResources
}
@ -715,7 +738,7 @@ func getDynamicValue(value string, features *nfdv1alpha1.Features) (string, erro
return element, nil
}
func filterTaints(taints []corev1.Taint) []corev1.Taint {
func (m *nfdMaster) filterTaints(taints []corev1.Taint) []corev1.Taint {
outTaints := []corev1.Taint{}
for _, taint := range taints {
@ -726,6 +749,12 @@ func filterTaints(taints []corev1.Taint) []corev1.Taint {
outTaints = append(outTaints, taint)
}
}
if m.config.Restrictions.MaxTaintsPerCR > 0 && len(taints) > m.config.Restrictions.MaxTaintsPerCR {
klog.InfoS("number of taints in the request exceeds the restriction maximum taints per CR, skipping")
outTaints = []corev1.Taint{}
}
return outTaints
}
@ -1031,13 +1060,24 @@ func (m *nfdMaster) refreshNodeFeatures(cli k8sclient.Interface, node *corev1.No
maps.Copy(extendedResources, crExtendedResources)
extendedResources = m.filterExtendedResources(features, extendedResources)
if m.config.Restrictions.MaxExtendedResourcesPerCR > 0 && len(extendedResources) > m.config.Restrictions.MaxExtendedResourcesPerCR {
klog.InfoS("number of extended resources in the request exceeds the restriction maximum extended resources per CR, skipping")
extendedResources = map[string]string{}
}
// Annotations
annotations := m.filterFeatureAnnotations(crAnnotations)
// Taints
var taints []corev1.Taint
if m.config.EnableTaints {
taints = filterTaints(crTaints)
taints = m.filterTaints(crTaints)
}
// If we deny node feature labels, we'll empty the labels variable
if m.config.Restrictions.DenyNodeFeatureLabels {
klog.InfoS("node feature labels are denied, skipping...")
labels = map[string]string{}
}
if m.config.NoPublish {
@ -1114,7 +1154,7 @@ func setTaints(cli k8sclient.Interface, taints []corev1.Taint, node *corev1.Node
newAnnotations[nfdv1alpha1.NodeTaintsAnnotation] = strings.Join(taintStrs, ",")
}
patches := createPatches([]string{nfdv1alpha1.NodeTaintsAnnotation}, node.Annotations, newAnnotations, "/metadata/annotations")
patches := createPatches([]string{nfdv1alpha1.NodeTaintsAnnotation}, node.Annotations, newAnnotations, "/metadata/annotations", true)
if len(patches) > 0 {
if err := patchNode(cli, node.Name, patches); err != nil {
return fmt.Errorf("error while patching node object: %w", err)
@ -1254,7 +1294,7 @@ func (m *nfdMaster) updateNodeObject(cli k8sclient.Interface, node *corev1.Node,
// Create JSON patches for changes in labels and annotations
oldLabels := stringToNsNames(node.Annotations[m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation)], nfdv1alpha1.FeatureLabelNs)
oldAnnotations := stringToNsNames(node.Annotations[m.instanceAnnotation(nfdv1alpha1.FeatureAnnotationsTrackingAnnotation)], nfdv1alpha1.FeatureAnnotationNs)
patches := createPatches(oldLabels, node.Labels, labels, "/metadata/labels")
patches := createPatches(oldLabels, node.Labels, labels, "/metadata/labels", m.config.Restrictions.OverwriteLabels)
oldAnnotations = append(oldAnnotations, []string{
m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation),
m.instanceAnnotation(nfdv1alpha1.ExtendedResourceAnnotation),
@ -1262,7 +1302,7 @@ func (m *nfdMaster) updateNodeObject(cli k8sclient.Interface, node *corev1.Node,
// Clean up deprecated/stale nfd version annotations
m.instanceAnnotation(nfdv1alpha1.MasterVersionAnnotation),
m.instanceAnnotation(nfdv1alpha1.WorkerVersionAnnotation)}...)
patches = append(patches, createPatches(oldAnnotations, node.Annotations, annotations, "/metadata/annotations")...)
patches = append(patches, createPatches(oldAnnotations, node.Annotations, annotations, "/metadata/annotations", true)...)
// patch node status with extended resource changes
statusPatches := m.createExtendedResourcePatches(node, extendedResources)
@ -1294,7 +1334,7 @@ func (m *nfdMaster) updateNodeObject(cli k8sclient.Interface, node *corev1.Node,
}
// createPatches is a generic helper that returns json patch operations to perform
func createPatches(removeKeys []string, oldItems map[string]string, newItems map[string]string, jsonPath string) []utils.JsonPatch {
func createPatches(removeKeys []string, oldItems map[string]string, newItems map[string]string, jsonPath string, overwrite bool) []utils.JsonPatch {
patches := []utils.JsonPatch{}
// Determine items to remove
@ -1309,7 +1349,7 @@ func createPatches(removeKeys []string, oldItems map[string]string, newItems map
// Determine items to add or replace
for key, newVal := range newItems {
if oldVal, ok := oldItems[key]; ok {
if newVal != oldVal {
if newVal != oldVal && overwrite {
patches = append(patches, utils.NewJsonPatch("replace", jsonPath, key, newVal))
}
} else {
@ -1511,8 +1551,10 @@ func (m *nfdMaster) startNfdApiController() error {
}
klog.InfoS("starting the nfd api controller")
m.nfdController, err = newNfdController(kubeconfig, nfdApiControllerOptions{
DisableNodeFeature: !nfdfeatures.NFDFeatureGate.Enabled(nfdfeatures.NodeFeatureAPI),
ResyncPeriod: m.config.ResyncPeriod.Duration,
DisableNodeFeature: !nfdfeatures.NFDFeatureGate.Enabled(nfdfeatures.NodeFeatureAPI),
ResyncPeriod: m.config.ResyncPeriod.Duration,
K8sClient: m.k8sClient,
NodeFeatureNamespaceSelector: m.config.Restrictions.AllowedNamespaces,
})
if err != nil {
return fmt.Errorf("failed to initialize CRD controller: %w", err)

221
test/e2e/node_feature_discovery_test.go
View file

@ -853,6 +853,227 @@ core:
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).WithTimeout(1 * time.Minute).Should(MatchLabels(expectedLabels, nodes))
})
Context("allowed namespaces restriction is respected or not", func() {
BeforeEach(func(ctx context.Context) {
extraMasterPodSpecOpts = []testpod.SpecOption{
testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"),
}
cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", `
restrictions:
allowedNamespaces:
matchLabels:
kubernetes.io/metadata.name: "fake"
`)
_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
})
It("Nothing should be created", func(ctx context.Context) {
// deploy node feature object
if !useNodeFeatureApi {
Skip("NodeFeature API not enabled")
}
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
Expect(err).NotTo(HaveOccurred())
targetNodeName := nodes[0].Name
Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
// Apply Node Feature object
By("Creating NodeFeature object")
nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(ctx, nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName)
Expect(err).NotTo(HaveOccurred())
By("Verifying node labels from NodeFeature object #1 are not created")
// No labels should be created since the f.Namespace is not in the allowed Namespaces
expectedLabels := map[string]k8sLabels{}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))
By("Deleting NodeFeature object")
err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
})
})
Context("max labels, taints restrictions should be respected", func() {
BeforeEach(func(ctx context.Context) {
extraMasterPodSpecOpts = []testpod.SpecOption{
testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"),
}
cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", `
enableTaints: true
restrictions:
maxLabelsPerCR: 1
maxTaintsPerCR: 1
`)
_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
})
It("No labels or taints should be created", func(ctx context.Context) {
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
Expect(err).NotTo(HaveOccurred())
targetNodeName := nodes[0].Name
Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
By("Creating nfd-worker config")
cm := testutils.NewConfigMap("nfd-worker-conf", "nfd-worker.conf", `
core:
sleepInterval: "1s"
featureSources: ["fake"]
labelSources: []
`)
cm, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
By("Creating nfd-worker daemonset")
podSpecOpts := createPodSpecOpts(
testpod.SpecWithContainerImage(dockerImage()),
testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"),
)
workerDS := testds.NFDWorker(podSpecOpts...)
workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(ctx, workerDS, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
By("Waiting for worker daemonset pods to be ready")
Expect(testpod.WaitForReady(ctx, f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 2)).NotTo(HaveOccurred())
// Add features from NodeFeatureRule #3
By("Creating NodeFeatureRules #3")
Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-3.yaml")).NotTo(HaveOccurred())
By("Verifying node taints and annotation from NodeFeatureRules #3")
expectedLabels := map[string]k8sLabels{}
expectedTaints := map[string][]corev1.Taint{}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes))
By("Deleting NodeFeatureRule #3")
err = nfdClient.NfdV1alpha1().NodeFeatureRules().Delete(ctx, "e2e-test-3", metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
By("Verifying taints from NodeFeatureRules #3 were removed")
expectedTaints["*"] = []corev1.Taint{}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes))
})
})
Context("max extended resources restriction should be respected", func() {
BeforeEach(func(ctx context.Context) {
extraMasterPodSpecOpts = []testpod.SpecOption{
testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"),
}
cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", `
restrictions:
maxExtendedResourcesPerCR: 1
`)
_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
})
It("Nothing should be created", func(ctx context.Context) {
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
Expect(err).NotTo(HaveOccurred())
targetNodeName := nodes[0].Name
Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
By("Creating nfd-worker config")
cm := testutils.NewConfigMap("nfd-worker-conf", "nfd-worker.conf", `
core:
sleepInterval: "1s"
featureSources: ["fake"]
labelSources: []
`)
cm, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
By("Creating nfd-worker daemonset")
podSpecOpts := createPodSpecOpts(
testpod.SpecWithContainerImage(dockerImage()),
testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"),
)
workerDS := testds.NFDWorker(podSpecOpts...)
workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(ctx, workerDS, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
By("Waiting for worker daemonset pods to be ready")
Expect(testpod.WaitForReady(ctx, f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 2)).NotTo(HaveOccurred())
expectedAnnotations := map[string]k8sAnnotations{
"*": {},
}
expectedCapacity := map[string]corev1.ResourceList{
"*": {},
}
By("Creating NodeFeatureRules #4")
Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-4.yaml")).NotTo(HaveOccurred())
By("Verifying node annotations from NodeFeatureRules #4")
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))
By("Verifying node status capacity from NodeFeatureRules #4")
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).WithTimeout(1 * time.Minute).Should(MatchCapacity(expectedCapacity, nodes))
By("Deleting NodeFeatureRules #4")
err = nfdClient.NfdV1alpha1().NodeFeatureRules().Delete(ctx, "e2e-extened-resource-test", metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
By("Deleting nfd-worker daemonset")
err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(ctx, workerDS.Name, metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
if useNodeFeatureApi {
By("Verify that labels from nfd-worker are garbage-collected")
expectedLabels := map[string]k8sLabels{
"*": {},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).WithTimeout(1 * time.Minute).Should(MatchLabels(expectedLabels, nodes))
}
})
})
Context("deny node feature labels restriction should be respected", func() {
BeforeEach(func(ctx context.Context) {
extraMasterPodSpecOpts = []testpod.SpecOption{
testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"),
}
cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", `
restrictions:
denyNodeFeatureLabels: true
`)
_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
})
It("No labels should be created", func(ctx context.Context) {
// deploy node feature object
if !useNodeFeatureApi {
Skip("NodeFeature API not enabled")
}
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
Expect(err).NotTo(HaveOccurred())
targetNodeName := nodes[0].Name
Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
// Apply Node Feature object
By("Creating NodeFeature object")
nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(ctx, nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName)
Expect(err).NotTo(HaveOccurred())
By("Verifying node labels from NodeFeature object #1 are not created")
expectedLabels := map[string]k8sLabels{
"*": {},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))
By("Deleting NodeFeature object")
err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
})
})
})
// Test NodeFeatureGroups