mirrors/node-feature-discovery
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/node-feature-discovery.git synced 2024-12-14 11:57:51 +00:00
Code Activity

Merge pull request #1592 from AhmedThresh/feat-configure-cr-restrictions

Browse source 
feat/nfd-master: configure CR restrictions
This commit is contained in:
Kubernetes Prow Robot 2024-10-24 12:20:54 +01:00 committed by GitHub
commit fd2893e2a5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 728 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
deployment/base/rbac/master-clusterrole.yaml
View file

@ -3,6 +3,13 @@ kind: ClusterRole
metadata:
name: nfd-master
rules:
- apiGroups:
- ""
resources:
- namespaces
verbs:
- watch
- list
- apiGroups:
- ""
resources:

15
deployment/components/master-config/nfd-master.conf.example
View file

@ -6,6 +6,21 @@
# enableTaints: false
# labelWhiteList: "foo"
# resyncPeriod: "2h"
# restrictions:
# disableLabels: true
# disableTaints: true
# disableExtendedResources: true
# disableAnnotations: true
# allowOverwrite: false
# denyNodeFeatureLabels: true
# nodeFeatureNamespaceSelector:
# matchLabels:
# kubernetes.io/metadata.name: "node-feature-discovery"
# matchExpressions:
# - key: "kubernetes.io/metadata.name"
# operator: "In"
# values:
# - "node-feature-discovery"
# klog:
# addDirHeader: false
# alsologtostderr: false

7
deployment/helm/node-feature-discovery/templates/clusterrole.yaml
View file

@ -6,6 +6,13 @@ metadata:
labels:
{{- include "node-feature-discovery.labels" . | nindent 4 }}
rules:
- apiGroups:
- ""
resources:
- namespaces
verbs:
- watch
- list
- apiGroups:
- ""
resources:

15
deployment/helm/node-feature-discovery/values.yaml
View file

@ -30,6 +30,21 @@ master:
# enableTaints: false
# labelWhiteList: "foo"
# resyncPeriod: "2h"
# restrictions:
# disableLabels: true
# disableTaints: true
# disableExtendedResources: true
# disableAnnotations: true
# allowOverwrite: false
# denyNodeFeatureLabels: true
# nodeFeatureNamespaceSelector:
# matchLabels:
# kubernetes.io/metadata.name: "node-feature-discovery"
# matchExpressions:
# - key: "kubernetes.io/metadata.name"
# operator: "In"
# values:
# - "node-feature-discovery"
# klog:
# addDirHeader: false
# alsologtostderr: false

101
docs/reference/master-configuration-reference.md
View file

@ -338,3 +338,104 @@ Comma-separated list of `pattern=N` settings for file-filtered logging.
Default: *empty*
Run-time configurable: yes
## restrictions (EXPERIMENTAL)
The following options specify the restrictions that can be applied by the
nfd-master on the deployed Custom Resources in the cluster.
### restrictions.nodeFeatureNamespaceSelector
The `nodeFeatureNamespaceSelector` option specifies the NodeFeatures namespaces
to watch, which can be selected by using `metav1.LabelSelector` as a type for
this option. An empty value selects all namespaces to be watched.
Default: *empty*
Example:
```yaml
restrictions:
nodeFeatureNamespaceSelector:
matchLabels:
kubernetes.io/metadata.name: "node-feature-discovery"
matchExpressions:
- key: "kubernetes.io/metadata.name"
operator: "In"
values:
- "node-feature-discovery"
```
### restrictions.disableLabels
The `disableLabels` option controls whether to allow creation of node labels
from NodeFeature and NodeFeatureRule CRs or not.
Default: false
Example:
```yaml
restrictions:
disableLabels: true
```
### restrictions.disableExtendedResources
The `disableExtendedResources` option controls whether to allow creation of
node extended resources from NodeFeatureRule CR or not.
Default: false
Example:
```yaml
restrictions:
disableExtendedResources: true
```
### restrictions.disableAnnotations
he `disableAnnotations` option controls whether to allow creation of node annotations
from NodeFeatureRule CR or not.
Default: false
Example:
```yaml
restrictions:
disableAnnotations: true
```
### restrictions.allowOverwrite
The `allowOverwrite` option controls whether NFD is allowed to overwrite and
take over management of existing node labels, annotations, and extended resources.
Labels, annotations and extended resources created by NFD itself are not affected
(overwrite cannot be disabled). NFD tracks the labels, annotations and extended
resources that it manages with specific
[node annotations](../get-started/introduction.md#node-annotations).
Default: true
Example:
```yaml
restrictions:
allowOverwrite: false
```
### restrictions.denyNodeFeatureLabels
The `denyNodeFeatureLabels` option specifies whether to deny labels from 3rd party
NodeFeature objects or not. NodeFeature objects created by nfd-worker are not affected.
Default: false
Example:
```yaml
restrictions:
denyNodeFeatureLabels: true
```

58
pkg/nfd-master/namespace-lister.go Normal file
View file

@ -0,0 +1,58 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nfdmaster
import (
"time"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/informers"
k8sclient "k8s.io/client-go/kubernetes"
v1lister "k8s.io/client-go/listers/core/v1"
)
// NamespaceLister lists kubernetes namespaces.
type NamespaceLister struct {
namespaceLister v1lister.NamespaceLister
labelsSelector labels.Selector
stopChan chan struct{}
}
func newNamespaceLister(k8sClient k8sclient.Interface, labelsSelector labels.Selector) *NamespaceLister {
factory := informers.NewSharedInformerFactory(k8sClient, time.Hour)
namespaceLister := factory.Core().V1().Namespaces().Lister()
stopChan := make(chan struct{})
factory.Start(stopChan) // runs in background
factory.WaitForCacheSync(stopChan)
return &NamespaceLister{
namespaceLister: namespaceLister,
labelsSelector: labelsSelector,
stopChan: stopChan,
}
}
// list returns all kubernetes namespaces.
func (lister *NamespaceLister) list() ([]*corev1.Namespace, error) {
return lister.namespaceLister.List(lister.labelsSelector)
}
// stop closes the channel used by the lister
func (lister *NamespaceLister) stop() {
close(lister.stopChan)
}

50
pkg/nfd-master/nfd-api-controller.go
View file

@ -22,6 +22,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
k8sclient "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
@ -46,12 +47,16 @@ type nfdController struct {
updateOneNodeChan chan string
updateAllNodeFeatureGroupsChan chan struct{}
updateNodeFeatureGroupChan chan string
namespaceLister *NamespaceLister
}
type nfdApiControllerOptions struct {
DisableNodeFeature bool
DisableNodeFeatureGroup bool
ResyncPeriod time.Duration
DisableNodeFeature bool
DisableNodeFeatureGroup bool
ResyncPeriod time.Duration
K8sClient k8sclient.Interface
NodeFeatureNamespaceSelector *metav1.LabelSelector
}
func init() {
@ -67,8 +72,16 @@ func newNfdController(config *restclient.Config, nfdApiControllerOptions nfdApiC
updateNodeFeatureGroupChan: make(chan string),
}
nfdClient := nfdclientset.NewForConfigOrDie(config)
if nfdApiControllerOptions.NodeFeatureNamespaceSelector != nil {
labelMap, err := metav1.LabelSelectorAsSelector(nfdApiControllerOptions.NodeFeatureNamespaceSelector)
if err != nil {
klog.ErrorS(err, "failed to convert label selector to map", "selector", nfdApiControllerOptions.NodeFeatureNamespaceSelector)
return nil, err
}
c.namespaceLister = newNamespaceLister(nfdApiControllerOptions.K8sClient, labelMap)
}
nfdClient := nfdclientset.NewForConfigOrDie(config)
klog.V(2).InfoS("initializing new NFD API controller", "options", utils.DelayedDumper(nfdApiControllerOptions))
informerFactory := nfdinformers.NewSharedInformerFactory(nfdClient, nfdApiControllerOptions.ResyncPeriod)
@ -89,7 +102,11 @@ func newNfdController(config *restclient.Config, nfdApiControllerOptions nfdApiC
AddFunc: func(obj interface{}) {
nfr := obj.(*nfdv1alpha1.NodeFeature)
klog.V(2).InfoS("NodeFeature added", "nodefeature", klog.KObj(nfr))
c.updateOneNode("NodeFeature", nfr)
if c.isNamespaceSelected(nfr.Namespace) {
c.updateOneNode("NodeFeature", nfr)
} else {
klog.V(2).InfoS("NodeFeature namespace is not selected, skipping", "nodefeature", klog.KObj(nfr))
}
if !nfdApiControllerOptions.DisableNodeFeatureGroup {
c.updateAllNodeFeatureGroups()
}
@ -187,6 +204,7 @@ func newNfdController(config *restclient.Config, nfdApiControllerOptions nfdApiC
func (c *nfdController) stop() {
close(c.stopChan)
c.namespaceLister.stop()
}
func getNodeNameForObj(obj metav1.Object) (string, error) {
@ -212,6 +230,28 @@ func (c *nfdController) updateOneNode(typ string, obj metav1.Object) {
}
}
func (c *nfdController) isNamespaceSelected(namespace string) bool {
// this means that the user didn't specify any namespace selector
// which means that we allow all namespaces
if c.namespaceLister == nil {
return true
}
namespaces, err := c.namespaceLister.list()
if err != nil {
klog.ErrorS(err, "failed to query namespaces by the namespace lister")
return false
}
for _, ns := range namespaces {
if ns.Name == namespace {
return true
}
}
return false
}
func (c *nfdController) updateAllNodes() {
select {
case c.updateAllNodesChan <- struct{}{}:

69
pkg/nfd-master/nfd-api-controller_test.go
View file

@ -20,8 +20,12 @@ import (
"testing"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
fakeclient "k8s.io/client-go/kubernetes/fake"
clienttesting "k8s.io/client-go/testing"
nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/api/nfd/v1alpha1"
)
@ -42,3 +46,66 @@ func TestGetNodeNameForObj(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, n, "node-1")
}
func newTestNamespace(name string) *corev1.Namespace {
return &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: map[string]string{
"name": name,
},
},
}
}
func TestIsNamespaceSelected(t *testing.T) {
fakeCli := fakeclient.NewSimpleClientset(newTestNamespace("fake"))
fakeCli.PrependWatchReactor("*", func(action clienttesting.Action) (handled bool, ret watch.Interface, err error) {
gvr := action.GetResource()
ns := action.GetNamespace()
watch, err := fakeCli.Tracker().Watch(gvr, ns)
if err != nil {
return false, nil, err
}
return true, watch, nil
})
c := &nfdController{}
testcases := []struct {
name string
objectNamespace string
nodeFeatureNamespaceSelector *metav1.LabelSelector
expectedResult bool
}{
{
name: "namespace not selected",
objectNamespace: "random",
nodeFeatureNamespaceSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "name",
Operator: metav1.LabelSelectorOpIn,
Values: []string{"fake"},
},
},
},
expectedResult: false,
},
{
name: "namespace is selected",
objectNamespace: "fake",
nodeFeatureNamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"name": "fake"},
},
expectedResult: true,
},
}
for _, tc := range testcases {
labelMap, _ := metav1.LabelSelectorAsSelector(tc.nodeFeatureNamespaceSelector)
c.namespaceLister = newNamespaceLister(fakeCli, labelMap)
res := c.isNamespaceSelected(tc.objectNamespace)
assert.Equal(t, res, tc.expectedResult)
}
}

31
pkg/nfd-master/nfd-master-internal_test.go
View file

@ -34,6 +34,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
fakeclient "k8s.io/client-go/kubernetes/fake"
fakecorev1client "k8s.io/client-go/kubernetes/typed/core/v1/fake"
clienttesting "k8s.io/client-go/testing"
@ -111,7 +112,7 @@ func withConfig(config *NFDConfig) NfdMasterOption {
func newFakeMaster(opts ...NfdMasterOption) *nfdMaster {
defaultOpts := []NfdMasterOption{
withNodeName(testNodeName),
withConfig(&NFDConfig{}),
withConfig(&NFDConfig{Restrictions: Restrictions{AllowOverwrite: true}}),
WithKubernetesClient(fakeclient.NewSimpleClientset()),
}
m, err := NewNfdMaster(append(defaultOpts, opts...)...)
@ -508,15 +509,16 @@ func TestFilterLabels(t *testing.T) {
func TestCreatePatches(t *testing.T) {
Convey("When creating JSON patches", t, func() {
existingItems := map[string]string{"key-1": "val-1", "key-2": "val-2", "key-3": "val-3"}
overwriteKeys := true
jsonPath := "/root"
Convey("When when there are neither itmes to remoe nor to add or update", func() {
p := createPatches([]string{"foo", "bar"}, existingItems, map[string]string{}, jsonPath)
Convey("When there are neither itmes to remoe nor to add or update", func() {
p := createPatches(sets.New([]string{"foo", "bar"}...), existingItems, map[string]string{}, jsonPath, overwriteKeys)
So(len(p), ShouldEqual, 0)
})
Convey("When when there are itmes to remoe but none to add or update", func() {
p := createPatches([]string{"key-2", "key-3", "foo"}, existingItems, map[string]string{}, jsonPath)
Convey("When there are itmes to remoe but none to add or update", func() {
p := createPatches(sets.New([]string{"key-2", "key-3", "foo"}...), existingItems, map[string]string{}, jsonPath, overwriteKeys)
expected := []utils.JsonPatch{
utils.NewJsonPatch("remove", jsonPath, "key-2", ""),
utils.NewJsonPatch("remove", jsonPath, "key-3", ""),
@ -524,9 +526,9 @@ func TestCreatePatches(t *testing.T) {
So(sortJsonPatches(p), ShouldResemble, sortJsonPatches(expected))
})
Convey("When when there are no itmes to remove but new items to add", func() {
Convey("When there are no itmes to remove but new items to add", func() {
newItems := map[string]string{"new-key": "new-val", "key-1": "new-1"}
p := createPatches([]string{"key-1"}, existingItems, newItems, jsonPath)
p := createPatches(sets.New([]string{"key-1"}...), existingItems, newItems, jsonPath, overwriteKeys)
expected := []utils.JsonPatch{
utils.NewJsonPatch("add", jsonPath, "new-key", newItems["new-key"]),
utils.NewJsonPatch("replace", jsonPath, "key-1", newItems["key-1"]),
@ -534,9 +536,9 @@ func TestCreatePatches(t *testing.T) {
So(sortJsonPatches(p), ShouldResemble, sortJsonPatches(expected))
})
Convey("When when there are items to remove add and update", func() {
Convey("When there are items to remove add and update", func() {
newItems := map[string]string{"new-key": "new-val", "key-2": "new-2", "key-4": "val-4"}
p := createPatches([]string{"key-1", "key-2", "key-3", "foo"}, existingItems, newItems, jsonPath)
p := createPatches(sets.New([]string{"key-1", "key-2", "key-3", "foo"}...), existingItems, newItems, jsonPath, overwriteKeys)
expected := []utils.JsonPatch{
utils.NewJsonPatch("add", jsonPath, "new-key", newItems["new-key"]),
utils.NewJsonPatch("add", jsonPath, "key-4", newItems["key-4"]),
@ -546,6 +548,17 @@ func TestCreatePatches(t *testing.T) {
}
So(sortJsonPatches(p), ShouldResemble, sortJsonPatches(expected))
})
Convey("When overwrite of keys is denied and there is already an existant key", func() {
overwriteKeys = false
newItems := map[string]string{"key-1": "new-2", "key-4": "val-4"}
p := createPatches(sets.New([]string{}...), existingItems, newItems, jsonPath, overwriteKeys)
expected := []utils.JsonPatch{
utils.NewJsonPatch("add", jsonPath, "key-4", newItems["key-4"]),
utils.NewJsonPatch("replace", jsonPath, "key-1", newItems["key-1"]),
}
So(sortJsonPatches(p), ShouldResemble, sortJsonPatches(expected))
})
})
}

116
pkg/nfd-master/nfd-master.go
View file

@ -45,14 +45,13 @@ import (
"k8s.io/apimachinery/pkg/labels"
k8sLabels "k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
k8sclient "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
"k8s.io/klog/v2"
controller "k8s.io/kubernetes/pkg/controller"
klogutils "sigs.k8s.io/node-feature-discovery/pkg/utils/klog"
taintutils "k8s.io/kubernetes/pkg/util/taints"
"sigs.k8s.io/yaml"
@ -63,6 +62,7 @@ import (
nfdfeatures "sigs.k8s.io/node-feature-discovery/pkg/features"
pb "sigs.k8s.io/node-feature-discovery/pkg/labeler"
"sigs.k8s.io/node-feature-discovery/pkg/utils"
klogutils "sigs.k8s.io/node-feature-discovery/pkg/utils/klog"
"sigs.k8s.io/node-feature-discovery/pkg/version"
)
@ -75,6 +75,16 @@ type ExtendedResources map[string]string
// Annotations are used for NFD-related node metadata
type Annotations map[string]string
// Restrictions contains the restrictions on the NF and NFR Crs
type Restrictions struct {
NodeFeatureNamespaceSelector *metav1.LabelSelector
DisableLabels bool
DisableExtendedResources bool
DisableAnnotations bool
DenyNodeFeatureLabels bool
AllowOverwrite bool
}
// NFDConfig contains the configuration settings of NfdMaster.
type NFDConfig struct {
AutoDefaultNs bool
@ -88,6 +98,7 @@ type NFDConfig struct {
LeaderElection LeaderElectionConfig
NfdApiParallelism int
Klog klogutils.KlogConfigOpts
Restrictions Restrictions
}
// LeaderElectionConfig contains the configuration for leader election
@ -273,6 +284,13 @@ func newDefaultConfig() *NFDConfig {
RenewDeadline: utils.DurationVal{Duration: time.Duration(10) * time.Second},
},
Klog: make(map[string]string),
Restrictions: Restrictions{
DisableLabels: false,
DisableExtendedResources: false,
DisableAnnotations: false,
AllowOverwrite: true,
DenyNodeFeatureLabels: false,
},
}
}
@ -581,10 +599,10 @@ func (m *nfdMaster) updateMasterNode() error {
}
// Advertise NFD version as an annotation
p := createPatches([]string{m.instanceAnnotation(nfdv1alpha1.MasterVersionAnnotation)},
p := createPatches(sets.New([]string{m.instanceAnnotation(nfdv1alpha1.MasterVersionAnnotation)}...),
node.Annotations,
nil,
"/metadata/annotations")
"/metadata/annotations", m.config.Restrictions.AllowOverwrite)
err = patchNode(m.k8sClient, node.Name, p)
if err != nil {
@ -625,6 +643,11 @@ func (m *nfdMaster) filterFeatureLabels(labels Labels, features *nfdv1alpha1.Fea
}
}
if len(outLabels) > 0 && m.config.Restrictions.DisableLabels {
klog.V(2).InfoS("node labels are disabled in configuration (restrictions.disableLabels=true)")
outLabels = Labels{}
}
return outLabels, extendedResources
}
@ -690,6 +713,7 @@ func filterTaints(taints []corev1.Taint) []corev1.Taint {
outTaints = append(outTaints, taint)
}
}
return outTaints
}
@ -781,42 +805,62 @@ func (m *nfdMaster) getAndMergeNodeFeatures(nodeName string) (*nfdv1alpha1.NodeF
return &nfdv1alpha1.NodeFeature{}, fmt.Errorf("failed to get NodeFeature resources for node %q: %w", nodeName, err)
}
filteredObjs := []*nfdv1alpha1.NodeFeature{}
for _, obj := range objs {
if m.isNamespaceSelected(obj.Namespace) {
filteredObjs = append(filteredObjs, obj)
}
}
// Node without a running NFD-Worker
if len(objs) == 0 {
if len(filteredObjs) == 0 {
return &nfdv1alpha1.NodeFeature{}, nil
}
// Sort our objects
sort.Slice(objs, func(i, j int) bool {
sort.Slice(filteredObjs, func(i, j int) bool {
// Objects in our nfd namespace gets into the beginning of the list
if objs[i].Namespace == m.namespace && objs[j].Namespace != m.namespace {
if filteredObjs[i].Namespace == m.namespace && filteredObjs[j].Namespace != m.namespace {
return true
}
if objs[i].Namespace != m.namespace && objs[j].Namespace == m.namespace {
if filteredObjs[i].Namespace != m.namespace && filteredObjs[j].Namespace == m.namespace {
return false
}
// After the nfd namespace, sort objects by their name
if objs[i].Name != objs[j].Name {
return objs[i].Name < objs[j].Name
if filteredObjs[i].Name != filteredObjs[j].Name {
return filteredObjs[i].Name < filteredObjs[j].Name
}
// Objects with the same name are sorted by their namespace
return objs[i].Namespace < objs[j].Namespace
return filteredObjs[i].Namespace < filteredObjs[j].Namespace
})
if len(objs) > 0 {
if len(filteredObjs) > 0 {
// Merge in features
//
// NOTE: changing the rule api to support handle multiple objects instead
// of merging would probably perform better with lot less data to copy.
features := objs[0].Spec.DeepCopy()
features := filteredObjs[0].Spec.DeepCopy()
if m.config.Restrictions.DenyNodeFeatureLabels && m.isThirdPartyNodeFeature(*filteredObjs[0], nodeName, m.namespace) {
klog.V(2).InfoS("node feature labels are disabled in configuration (restrictions.denyNodeFeatureLabels=true)")
features.Labels = nil
}
if !nfdfeatures.NFDFeatureGate.Enabled(nfdfeatures.DisableAutoPrefix) && m.config.AutoDefaultNs {
features.Labels = addNsToMapKeys(features.Labels, nfdv1alpha1.FeatureLabelNs)
}
for _, o := range objs[1:] {
for _, o := range filteredObjs[1:] {
s := o.Spec.DeepCopy()
if m.config.Restrictions.DenyNodeFeatureLabels && m.isThirdPartyNodeFeature(*o, nodeName, m.namespace) {
klog.V(2).InfoS("node feature labels are disabled in configuration (restrictions.denyNodeFeatureLabels=true)")
s.Labels = nil
}
if !nfdfeatures.NFDFeatureGate.Enabled(nfdfeatures.DisableAutoPrefix) && m.config.AutoDefaultNs {
s.Labels = addNsToMapKeys(s.Labels, nfdv1alpha1.FeatureLabelNs)
}
s.MergeInto(features)
}
@ -829,6 +873,11 @@ func (m *nfdMaster) getAndMergeNodeFeatures(nodeName string) (*nfdv1alpha1.NodeF
return nodeFeatures, nil
}
// isThirdPartyNodeFeature determines whether a node feature is a third party one or created by nfd-worker
func (m *nfdMaster) isThirdPartyNodeFeature(nodeFeature nfdv1alpha1.NodeFeature, nodeName, namespace string) bool {
return nodeFeature.Namespace != namespace || nodeFeature.Name != nodeName
}
func (m *nfdMaster) nfdAPIUpdateOneNode(cli k8sclient.Interface, node *corev1.Node) error {
if m.nfdController == nil || m.nfdController.featureLister == nil {
return nil
@ -995,6 +1044,11 @@ func (m *nfdMaster) refreshNodeFeatures(cli k8sclient.Interface, node *corev1.No
maps.Copy(extendedResources, crExtendedResources)
extendedResources = m.filterExtendedResources(features, extendedResources)
if len(extendedResources) > 0 && m.config.Restrictions.DisableExtendedResources {
klog.V(2).InfoS("extended resources are disabled in configuration (restrictions.disableExtendedResources=true)")
extendedResources = map[string]string{}
}
// Annotations
annotations := m.filterFeatureAnnotations(crAnnotations)
@ -1021,8 +1075,8 @@ func (m *nfdMaster) refreshNodeFeatures(cli k8sclient.Interface, node *corev1.No
// setTaints sets node taints and annotations based on the taints passed via
// nodeFeatureRule custom resorce. If empty list of taints is passed, currently
// NFD owned taints and annotations are removed from the node.
func setTaints(cli k8sclient.Interface, taints []corev1.Taint, node *corev1.Node) error {
// De-serialize the taints annotation into corev1.Taint type for comparison below.
func (m *nfdMaster) setTaints(cli k8sclient.Interface, taints []corev1.Taint, node *corev1.Node) error {
// De-serialize the taints annotation into corev1.Taint type for comparision below.
var err error
oldTaints := []corev1.Taint{}
if val, ok := node.Annotations[nfdv1alpha1.NodeTaintsAnnotation]; ok {
@ -1078,7 +1132,11 @@ func setTaints(cli k8sclient.Interface, taints []corev1.Taint, node *corev1.Node
newAnnotations[nfdv1alpha1.NodeTaintsAnnotation] = strings.Join(taintStrs, ",")
}
patches := createPatches([]string{nfdv1alpha1.NodeTaintsAnnotation}, node.Annotations, newAnnotations, "/metadata/annotations")
patches := createPatches(sets.New([]string{nfdv1alpha1.NodeTaintsAnnotation}...),
node.Annotations, newAnnotations,
"/metadata/annotations",
m.config.Restrictions.AllowOverwrite,
)
if len(patches) > 0 {
if err := patchNode(cli, node.Name, patches); err != nil {
return fmt.Errorf("error while patching node object: %w", err)
@ -1218,7 +1276,7 @@ func (m *nfdMaster) updateNodeObject(cli k8sclient.Interface, node *corev1.Node,
// Create JSON patches for changes in labels and annotations
oldLabels := stringToNsNames(node.Annotations[m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation)], nfdv1alpha1.FeatureLabelNs)
oldAnnotations := stringToNsNames(node.Annotations[m.instanceAnnotation(nfdv1alpha1.FeatureAnnotationsTrackingAnnotation)], nfdv1alpha1.FeatureAnnotationNs)
patches := createPatches(oldLabels, node.Labels, labels, "/metadata/labels")
patches := createPatches(sets.New(oldLabels...), node.Labels, labels, "/metadata/labels", m.config.Restrictions.AllowOverwrite)
oldAnnotations = append(oldAnnotations, []string{
m.instanceAnnotation(nfdv1alpha1.FeatureLabelsAnnotation),
m.instanceAnnotation(nfdv1alpha1.ExtendedResourceAnnotation),
@ -1226,7 +1284,7 @@ func (m *nfdMaster) updateNodeObject(cli k8sclient.Interface, node *corev1.Node,
// Clean up deprecated/stale nfd version annotations
m.instanceAnnotation(nfdv1alpha1.MasterVersionAnnotation),
m.instanceAnnotation(nfdv1alpha1.WorkerVersionAnnotation)}...)
patches = append(patches, createPatches(oldAnnotations, node.Annotations, annotations, "/metadata/annotations")...)
patches = append(patches, createPatches(sets.New(oldAnnotations...), node.Annotations, annotations, "/metadata/annotations", m.config.Restrictions.AllowOverwrite)...)
// patch node status with extended resource changes
statusPatches := m.createExtendedResourcePatches(node, extendedResources)
@ -1249,7 +1307,7 @@ func (m *nfdMaster) updateNodeObject(cli k8sclient.Interface, node *corev1.Node,
}
// Set taints
err = setTaints(cli, taints, node)
err = m.setTaints(cli, taints, node)
if err != nil {
return err
}
@ -1258,11 +1316,11 @@ func (m *nfdMaster) updateNodeObject(cli k8sclient.Interface, node *corev1.Node,
}
// createPatches is a generic helper that returns json patch operations to perform
func createPatches(removeKeys []string, oldItems map[string]string, newItems map[string]string, jsonPath string) []utils.JsonPatch {
func createPatches(removeKeys sets.Set[string], oldItems map[string]string, newItems map[string]string, jsonPath string, overwrite bool) []utils.JsonPatch {
patches := []utils.JsonPatch{}
// Determine items to remove
for _, key := range removeKeys {
for key := range removeKeys {
if _, ok := oldItems[key]; ok {
if _, ok := newItems[key]; !ok {
patches = append(patches, utils.NewJsonPatch("remove", jsonPath, key, ""))
@ -1273,7 +1331,7 @@ func createPatches(removeKeys []string, oldItems map[string]string, newItems map
// Determine items to add or replace
for key, newVal := range newItems {
if oldVal, ok := oldItems[key]; ok {
if newVal != oldVal {
if newVal != oldVal && (!removeKeys.Has(key) || overwrite) {
patches = append(patches, utils.NewJsonPatch("replace", jsonPath, key, newVal))
}
} else {
@ -1475,8 +1533,10 @@ func (m *nfdMaster) startNfdApiController() error {
}
klog.InfoS("starting the nfd api controller")
m.nfdController, err = newNfdController(kubeconfig, nfdApiControllerOptions{
DisableNodeFeature: !nfdfeatures.NFDFeatureGate.Enabled(nfdfeatures.NodeFeatureAPI),
ResyncPeriod: m.config.ResyncPeriod.Duration,
DisableNodeFeature: !nfdfeatures.NFDFeatureGate.Enabled(nfdfeatures.NodeFeatureAPI),
ResyncPeriod: m.config.ResyncPeriod.Duration,
K8sClient: m.k8sClient,
NodeFeatureNamespaceSelector: m.config.Restrictions.NodeFeatureNamespaceSelector,
})
if err != nil {
return fmt.Errorf("failed to initialize CRD controller: %w", err)
@ -1537,6 +1597,12 @@ func (m *nfdMaster) filterFeatureAnnotations(annotations map[string]string) map[
outAnnotations[annotation] = value
}
if len(outAnnotations) > 0 && m.config.Restrictions.DisableAnnotations {
klog.V(2).InfoS("node annotations are disabled in configuration (restrictions.disableAnnotations=true)")
outAnnotations = map[string]string{}
}
return outAnnotations
}

18
test/e2e/data/nodefeaturerule-6.yaml Normal file
View file

@ -0,0 +1,18 @@
apiVersion: nfd.k8s-sigs.io/v1alpha1
kind: NodeFeatureRule
metadata:
name: e2e-test-6
spec:
rules:
- name: "e2e-restrictions-test-1"
taints:
- effect: PreferNoSchedule
key: "feature.node.kubernetes.io/fake-special-cpu"
value: "true"
labels:
e2e.feature.node.kubernetes.io/restricted-label-1: "true"
annotations:
e2e.feature.node.kubernetes.io/restricted-annoation-1: "yes"
extendedResources:
e2e.feature.node.kubernetes.io/restricted-er-1: "2"
matchFeatures:

277
test/e2e/node_feature_discovery_test.go
View file

@ -28,7 +28,6 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
@ -994,6 +993,282 @@ resyncPeriod: "1s"
Expect(err).NotTo(HaveOccurred())
})
})
Context("selected namespaces restriction is respected or not", Label("restrictions"), func() {
BeforeEach(func(ctx context.Context) {
extraMasterPodSpecOpts = []testpod.SpecOption{
testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"),
}
cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", `
restrictions:
nodeFeatureNamespaceSelector:
matchLabels:
e2etest: fake
resyncPeriod: "1s"
`)
_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
})
It("Nothing should be created", func(ctx context.Context) {
// deploy node feature object
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
Expect(err).NotTo(HaveOccurred())
targetNodeName := nodes[0].Name
Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
// label the namespace in which node feature object is created
// TODO(TessaIO): add a utility for this.
patches, err := json.Marshal(
[]utils.JsonPatch{
utils.NewJsonPatch(
"add",
"/metadata/labels",
"e2etest",
"fake",
),
},
)
Expect(err).NotTo(HaveOccurred())
_, err = f.ClientSet.CoreV1().Namespaces().Patch(ctx, f.Namespace.Name, types.JSONPatchType, patches, metav1.PatchOptions{})
Expect(err).NotTo(HaveOccurred())
// Apply Node Feature object
By("Creating NodeFeature object")
nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(ctx, nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName)
Expect(err).NotTo(HaveOccurred())
By("Verifying node labels from NodeFeature object #1 are created")
// No labels should be created since the f.Namespace is not in the selected Namespaces
expectedLabels := map[string]k8sLabels{
targetNodeName: {
nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-1": "obj-1",
nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-2": "obj-1",
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "overridden",
},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))
// remove label the namespace in which node feature object is created
patches, err = json.Marshal(
[]utils.JsonPatch{
utils.NewJsonPatch(
"remove",
"/metadata/labels",
"e2etest",
"fake",
),
},
)
Expect(err).NotTo(HaveOccurred())
_, err = f.ClientSet.CoreV1().Namespaces().Patch(ctx, f.Namespace.Name, types.JSONPatchType, patches, metav1.PatchOptions{})
Expect(err).NotTo(HaveOccurred())
By("Verifying node labels from NodeFeature object #1 are not created")
// No labels should be created since the f.Namespace is not in the selected Namespaces
expectedLabels = map[string]k8sLabels{
targetNodeName: {},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))
By("Deleting NodeFeature object")
err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
})
})
Context("disable labels restrictions should be respected", Label("restrictions"), func() {
BeforeEach(func(ctx context.Context) {
extraMasterPodSpecOpts = []testpod.SpecOption{
testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"),
testpod.SpecWithContainerExtraArgs("-enable-taints"),
}
cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", `
restrictions:
disableLabels: true
`)
_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
})
It("No labels should be created", func(ctx context.Context) {
// deploy node feature object
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
Expect(err).NotTo(HaveOccurred())
// Add features from NodeFeatureRule #6
By("Creating NodeFeatureRules #6")
Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-6.yaml")).NotTo(HaveOccurred())
By("Verifying node taints, annotations, ERs and labels from NodeFeatureRules #6")
expectedTaints := map[string][]corev1.Taint{
"*": {
{
Key: "feature.node.kubernetes.io/fake-special-cpu",
Value: "true",
Effect: "PreferNoSchedule",
},
},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes))
expectedAnnotations := map[string]k8sAnnotations{
"*": {
"e2e.feature.node.kubernetes.io/restricted-annoation-1": "yes",
"nfd.node.kubernetes.io/feature-annotations": "e2e.feature.node.kubernetes.io/restricted-annoation-1",
"nfd.node.kubernetes.io/extended-resources": "e2e.feature.node.kubernetes.io/restricted-er-1",
"nfd.node.kubernetes.io/taints": "feature.node.kubernetes.io/fake-special-cpu=true:PreferNoSchedule",
},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))
expectedCapacity := map[string]corev1.ResourceList{
"*": {
"e2e.feature.node.kubernetes.io/restricted-er-1": resourcev1.MustParse("2"),
},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).WithTimeout(1 * time.Minute).Should(MatchCapacity(expectedCapacity, nodes))
expectedLabels := map[string]k8sLabels{
"*": {},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))
By("Deleting NodeFeatureRule #6")
err = nfdClient.NfdV1alpha1().NodeFeatureRules().Delete(ctx, "e2e-test-6", metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
})
})
Context("disable extended resources restriction should be respected", Label("restrictions"), func() {
BeforeEach(func(ctx context.Context) {
extraMasterPodSpecOpts = []testpod.SpecOption{
testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"),
}
cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", `
restrictions:
disableExtendedResources: true
`)
_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
})
It("Extended resources should not be created and Labels should be created", func(ctx context.Context) {
// deploy node feature object
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
Expect(err).NotTo(HaveOccurred())
targetNodeName := nodes[0].Name
Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
expectedAnnotations := map[string]k8sAnnotations{
"*": {
"e2e.feature.node.kubernetes.io/restricted-annoation-1": "yes",
"nfd.node.kubernetes.io/feature-annotations": "e2e.feature.node.kubernetes.io/restricted-annoation-1",
"nfd.node.kubernetes.io/feature-labels": "e2e.feature.node.kubernetes.io/restricted-label-1",
},
}
expectedCapacity := map[string]corev1.ResourceList{
"*": {},
}
expectedLabels := map[string]k8sLabels{
"*": {
"e2e.feature.node.kubernetes.io/restricted-label-1": "true",
},
}
By("Creating NodeFeatureRules #6")
Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-6.yaml")).NotTo(HaveOccurred())
By("Verifying node labels from NodeFeatureRules #6")
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))
By("Verifying node annotations from NodeFeatureRules #6")
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))
By("Verifying node status capacity from NodeFeatureRules #6")
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).WithTimeout(1 * time.Minute).Should(MatchCapacity(expectedCapacity, nodes))
By("Deleting NodeFeatureRules #6")
err = nfdClient.NfdV1alpha1().NodeFeatureRules().Delete(ctx, "e2e-test-6", metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
By("Verify that labels from nfd-worker are garbage-collected")
expectedLabels = map[string]k8sLabels{
"*": {},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).WithTimeout(1 * time.Minute).Should(MatchLabels(expectedLabels, nodes))
})
})
Context("deny node feature labels restriction should be respected", Label("restrictions"), func() {
BeforeEach(func(ctx context.Context) {
extraMasterPodSpecOpts = []testpod.SpecOption{
testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"),
}
cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", `
restrictions:
denyNodeFeatureLabels: true
`)
_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
})
It("No feature labels should be created", func(ctx context.Context) {
// deploy node feature object
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
Expect(err).NotTo(HaveOccurred())
targetNodeName := nodes[0].Name
Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
// Apply Node Feature object
By("Creating NodeFeature object")
nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(ctx, nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName)
Expect(err).NotTo(HaveOccurred())
// Add features from NodeFeatureRule #6
By("Creating NodeFeatureRules #6")
Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-6.yaml")).NotTo(HaveOccurred())
By("Verifying node taints and labels from NodeFeatureRules #6")
expectedTaints := map[string][]corev1.Taint{
"*": {},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes))
expectedAnnotations := map[string]k8sAnnotations{
"*": {
"e2e.feature.node.kubernetes.io/restricted-annoation-1": "yes",
"nfd.node.kubernetes.io/feature-annotations": "e2e.feature.node.kubernetes.io/restricted-annoation-1",
"nfd.node.kubernetes.io/extended-resources": "e2e.feature.node.kubernetes.io/restricted-er-1",
"nfd.node.kubernetes.io/feature-labels": "e2e.feature.node.kubernetes.io/restricted-label-1",
},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))
expectedCapacity := map[string]corev1.ResourceList{
"*": {
"e2e.feature.node.kubernetes.io/restricted-er-1": resourcev1.MustParse("2"),
},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).WithTimeout(1 * time.Minute).Should(MatchCapacity(expectedCapacity, nodes))
// TODO(TessaIO): we need one more test where we deploy nfd-worker that would create
// a non 3rd-party NF that shouldn't be ignored by this restriction
By("Verifying node labels from NodeFeature object #6 are not created")
expectedLabels := map[string]k8sLabels{
"*": {
"e2e.feature.node.kubernetes.io/restricted-label-1": "true",
},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))
By("Deleting NodeFeature object")
err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
})
})
})
})

5
test/e2e/utils/rbac.go
View file

@ -176,6 +176,11 @@ func createClusterRoleMaster(ctx context.Context, cs clientset.Interface) (*rbac
Name: "nfd-master-e2e",
},
Rules: []rbacv1.PolicyRule{
{
APIGroups: []string{""},
Resources: []string{"namespaces"},
Verbs: []string{"list", "watch"},
},
{
APIGroups: []string{""},
Resources: []string{"nodes", "nodes/status"},