From 7038e49d02965fde325656f50f9a082e4b1b90bf Mon Sep 17 00:00:00 2001 From: Marc Sluiter Date: Thu, 21 Jan 2021 00:48:40 +0100 Subject: [PATCH] source/custom: Add nodename rule There are cases when the only available metadata for discovering features is the node's name. The "nodename" rule extends the custom source and matches when the node's name matches one of the given nodename regexp patterns. It is also possible now to set an optional "value" on custom rules, which overrides the default "true" label value in case the rule matches. In order to allow more dynamic configurations without having to modify the complete worker configuration, custom rules are additionally read from a "custom.d" directory now. Typically that directory will be filled by mounting one or more ConfigMaps. Signed-off-by: Marc Sluiter --- deployment/node-feature-discovery/values.yaml | 4 + docs/get-started/features.md | 40 +++ nfd-daemonset-combined.yaml.template | 30 ++ nfd-worker-daemonset.yaml.template | 30 ++ nfd-worker-job.yaml.template | 30 ++ nfd-worker.conf.example | 4 + source/custom/custom.go | 85 +++--- source/custom/directory_features.go | 88 ++++++ source/custom/rules/nodename_rule.go | 50 ++++ test/e2e/node_feature_discovery.go | 277 ++++++++++++++---- 10 files changed, 534 insertions(+), 104 deletions(-) create mode 100644 source/custom/directory_features.go create mode 100644 source/custom/rules/nodename_rule.go diff --git a/deployment/node-feature-discovery/values.yaml b/deployment/node-feature-discovery/values.yaml index 29500851e..033a893e7 100644 --- a/deployment/node-feature-discovery/values.yaml +++ b/deployment/node-feature-discovery/values.yaml @@ -161,6 +161,10 @@ worker: # vendor: ["15b3"] # device: ["1014", "1017"] # loadedKMod : ["vendor_kmod1", "vendor_kmod2"] + # - name: "feature.by.nodename" + # value: customValue + # matchOn: + # - nodename: ["worker-0", "my-.*-node"] ### podSecurityContext: {} diff --git a/docs/get-started/features.md b/docs/get-started/features.md index e6d85248b..314892416 100644 --- a/docs/get-started/features.md +++ b/docs/get-started/features.md @@ -134,6 +134,17 @@ examples how to set-up and manage the worker configuration. To aid in making Custom Features clearer, we define a general and a per rule nomenclature, keeping things as consistent as possible. +#### Additional configuration directory + +Additionally to the rules defined in the nfd-worker configuration file, the +Custom feature can read more configuration files located in the +`/etc/kubernetes/node-feature-discovery/custom.d/` directory. This makes more +dynamic and flexible configuration easier. This directory must be available +inside the NFD worker container, so Volumes and VolumeMounts must be used for +mounting e.g. ConfigMap(s). The example deployment manifests provide an example +(commented out) for providing Custom configuration with an additional +ConfigMap, mounted into the `custom.d` directory. + #### General Nomenclature & Definitions ``` @@ -151,6 +162,7 @@ file. sources: custom: - name: + value: matchOn: - : [: ] @@ -291,6 +303,26 @@ Matching is done by performing logical _AND_ for each provided Element, i.e the Rule will match if all provided Elements (kernel config options) are enabled (`y` or `m`) or matching `=` in the kernel. +##### Nodename Rule + +###### Nomenclature + +``` +Element :A nodename regexp pattern +``` + +The Rule allows matching the node's name against a provided list of Elements. + +###### Format + +```yaml +nodename: [ , ... ] +``` + +Matching is done by performing logical _OR_ for each provided Element, i.e the +Rule will match if one of the provided Elements (nodename regexp pattern) +matches the node's name. + #### Example ```yaml @@ -328,6 +360,10 @@ custom: matchOn: - kConfig: ["GCC_VERSION=100101"] loadedKMod: ["kmod1"] + - name: "my.datacenter" + value: "datacenter-1" + matchOn: + - nodename: [ "node-datacenter1-rack.*-server.*" ] ``` __In the example above:__ @@ -360,6 +396,10 @@ __In the example above:__ `feature.node.kubernetes.io/custom-my.kernel.modulecompiler=true` if the in-tree `kmod1` kernel module is loaded __AND__ it's built with `GCC_VERSION=100101`. +- A node would contain the label: + `feature.node.kubernetes.io/my.datacenter=datacenter-1` if the node's name + matches the `node-datacenter1-rack.*-server.*` pattern, e.g. + `node-datacenter1-rack2-server42` #### Statically defined features diff --git a/nfd-daemonset-combined.yaml.template b/nfd-daemonset-combined.yaml.template index 2bb4f148f..7c14f7c70 100644 --- a/nfd-daemonset-combined.yaml.template +++ b/nfd-daemonset-combined.yaml.template @@ -115,6 +115,11 @@ spec: - name: nfd-worker-conf mountPath: "/etc/kubernetes/node-feature-discovery" readOnly: true +## Example for more custom configs in an additional configmap (1/3) +## Mounting into subdirectories of custom.d makes it easy to use multiple configmaps +# - name: custom-source-extra-rules +# mountPath: "/etc/kubernetes/node-feature-discovery/custom.d/extra-rules-1" +# readOnly: true volumes: - name: host-boot hostPath: @@ -134,6 +139,10 @@ spec: - name: nfd-worker-conf configMap: name: nfd-worker-conf +## Example for more custom configs in an additional configmap (2/3) +# - name: custom-source-extra-rules +# configMap: +# name: custom-source-extra-rules --- apiVersion: v1 kind: ConfigMap @@ -232,4 +241,25 @@ data: # vendor: ["15b3"] # device: ["1014", "1017"] # loadedKMod : ["vendor_kmod1", "vendor_kmod2"] + # - name: "feature.by.nodename" + # value: customValue + # matchOn: + # - nodename: ["worker-0", "my-.*-node"] ### +--- +## Example for more custom configs in an additional configmap (3/3) +#apiVersion: v1 +#kind: ConfigMap +#metadata: +# name: custom-source-extra-rules +# namespace: node-feature-discovery +#data: +## Filename doesn't matter, and there can be multiple. They just need to be unique. +# custom.conf: | +# - name: "more.kernel.features" +# matchOn: +# - loadedKMod: ["example_kmod3"] +# - name: "more.features.by.nodename" +# value: customValue +# matchOn: +# - nodename: ["special-.*-node-.*"] diff --git a/nfd-worker-daemonset.yaml.template b/nfd-worker-daemonset.yaml.template index ec04a4e52..d6ced4ba1 100644 --- a/nfd-worker-daemonset.yaml.template +++ b/nfd-worker-daemonset.yaml.template @@ -68,6 +68,11 @@ spec: - name: nfd-worker-conf mountPath: "/etc/kubernetes/node-feature-discovery" readOnly: true +## Example for more custom configs in an additional configmap (1/3) +## Mounting into subdirectories of custom.d makes it easy to use multiple configmaps +# - name: custom-source-extra-rules +# mountPath: "/etc/kubernetes/node-feature-discovery/custom.d/extra-rules-1" +# readOnly: true ## Enable TLS authentication (2/3) # - name: nfd-ca-cert # mountPath: "/etc/kubernetes/node-feature-discovery/trust" @@ -94,6 +99,10 @@ spec: - name: nfd-worker-conf configMap: name: nfd-worker-conf +## Example for more custom configs in an additional configmap (2/3) +# - name: custom-source-extra-rules +# configMap: +# name: custom-source-extra-rules ## Enable TLS authentication (3/3) # - name: nfd-ca-cert # configMap: @@ -199,4 +208,25 @@ data: # vendor: ["15b3"] # device: ["1014", "1017"] # loadedKMod : ["vendor_kmod1", "vendor_kmod2"] + # - name: "feature.by.nodename" + # value: customValue + # matchOn: + # - nodename: ["worker-0", "my-.*-node"] ### +--- +## Example for more custom configs in an additional configmap (3/3) +#apiVersion: v1 +#kind: ConfigMap +#metadata: +# name: custom-source-extra-rules +# namespace: node-feature-discovery +#data: +## Filename doesn't matter, and there can be multiple. They just need to be unique. +# custom.conf: | +# - name: "more.kernel.features" +# matchOn: +# - loadedKMod: ["example_kmod3"] +# - name: "more.features.by.nodename" +# value: customValue +# matchOn: +# - nodename: ["special-.*-node-.*"] diff --git a/nfd-worker-job.yaml.template b/nfd-worker-job.yaml.template index 6eaf9ec49..7fd8a0b65 100644 --- a/nfd-worker-job.yaml.template +++ b/nfd-worker-job.yaml.template @@ -72,6 +72,11 @@ spec: - name: nfd-worker-conf mountPath: "/etc/kubernetes/node-feature-discovery" readOnly: true +## Example for more custom configs in an additional configmap (1/3) +## Mounting into subdirectories of custom.d makes it easy to use multiple configmaps +# - name: custom-source-extra-rules +# mountPath: "/etc/kubernetes/node-feature-discovery/custom.d/extra-rules-1" +# readOnly: true ## Enable TLS authentication (2/3) # - name: nfd-ca-cert # mountPath: "/etc/kubernetes/node-feature-discovery/trust" @@ -99,6 +104,10 @@ spec: - name: nfd-worker-conf configMap: name: nfd-worker-conf +## Example for more custom configs in an additional configmap (2/3) +# - name: custom-source-extra-rules +# configMap: +# name: custom-source-extra-rules ## Enable TLS authentication (3/3) # - name: nfd-ca-cert # configMap: @@ -204,4 +213,25 @@ data: # vendor: ["15b3"] # device: ["1014", "1017"] # loadedKMod : ["vendor_kmod1", "vendor_kmod2"] + # - name: "feature.by.nodename" + # value: customValue + # matchOn: + # - nodename: ["worker-0", "my-.*-node"] ### +--- +## Example for more custom configs in an additional configmap (3/3) +#apiVersion: v1 +#kind: ConfigMap +#metadata: +# name: custom-source-extra-rules +# namespace: node-feature-discovery +#data: +## Filename doesn't matter, and there can be multiple. They just need to be unique. +# custom.conf: | +# - name: "more.kernel.features" +# matchOn: +# - loadedKMod: ["example_kmod3"] +# - name: "more.features.by.nodename" +# value: customValue +# matchOn: +# - nodename: ["special-.*-node-.*"] diff --git a/nfd-worker.conf.example b/nfd-worker.conf.example index 8f9447c1a..1f60ae521 100644 --- a/nfd-worker.conf.example +++ b/nfd-worker.conf.example @@ -88,3 +88,7 @@ # vendor: ["15b3"] # device: ["1014", "1017"] # loadedKMod : ["vendor_kmod1", "vendor_kmod2"] +# - name: "feature.by.nodename" +# value: customValue +# matchOn: +# - nodename: ["worker-0", "my-.*-node"] diff --git a/source/custom/custom.go b/source/custom/custom.go index f77e0dcd6..a49ad510e 100644 --- a/source/custom/custom.go +++ b/source/custom/custom.go @@ -18,6 +18,7 @@ package custom import ( "log" + "reflect" "sigs.k8s.io/node-feature-discovery/source" "sigs.k8s.io/node-feature-discovery/source/custom/rules" @@ -30,10 +31,12 @@ type MatchRule struct { LoadedKMod *rules.LoadedKModRule `json:"loadedKMod,omitempty"` CpuID *rules.CpuIDRule `json:"cpuId,omitempty"` Kconfig *rules.KconfigRule `json:"kConfig,omitempty"` + Nodename *rules.NodenameRule `json:"nodename,omitempty"` } type FeatureSpec struct { Name string `json:"name"` + Value *string `json:"value"` MatchOn []MatchRule `json:"matchOn"` } @@ -72,6 +75,7 @@ func (s *Source) SetConfig(conf source.Config) { func (s Source) Discover() (source.Features, error) { features := source.Features{} allFeatureConfig := append(getStaticFeatureConfig(), *s.config...) + allFeatureConfig = append(allFeatureConfig, getDirectoryFeatureConfig()...) log.Printf("INFO: Custom features: %+v", allFeatureConfig) // Iterate over features for _, customFeature := range allFeatureConfig { @@ -81,7 +85,11 @@ func (s Source) Discover() (source.Features, error) { continue } if featureExist { - features[customFeature.Name] = true + var value interface{} = true + if customFeature.Value != nil { + value = *customFeature.Value + } + features[customFeature.Name] = value } } return features, nil @@ -90,58 +98,37 @@ func (s Source) Discover() (source.Features, error) { // Process a single feature by Matching on the defined rules. // A feature is present if all defined Rules in a MatchRule return a match. func (s Source) discoverFeature(feature FeatureSpec) (bool, error) { - for _, rule := range feature.MatchOn { - // PCI ID rule - if rule.PciID != nil { - match, err := rule.PciID.Match() - if err != nil { - return false, err - } - if !match { - continue - } + for _, matchRules := range feature.MatchOn { + + allRules := []rules.Rule{ + matchRules.PciID, + matchRules.UsbID, + matchRules.LoadedKMod, + matchRules.CpuID, + matchRules.Kconfig, + matchRules.Nodename, } - // USB ID rule - if rule.UsbID != nil { - match, err := rule.UsbID.Match() - if err != nil { - return false, err - } - if !match { - continue + + // return true, nil if all rules match + matchRules := func(rules []rules.Rule) (bool, error) { + for _, rule := range rules { + if reflect.ValueOf(rule).IsNil() { + continue + } + if match, err := rule.Match(); err != nil { + return false, err + } else if !match { + return false, nil + } } + return true, nil } - // Loaded kernel module rule - if rule.LoadedKMod != nil { - match, err := rule.LoadedKMod.Match() - if err != nil { - return false, err - } - if !match { - continue - } + + if match, err := matchRules(allRules); err != nil { + return false, err + } else if match { + return true, nil } - // cpuid rule - if rule.CpuID != nil { - match, err := rule.CpuID.Match() - if err != nil { - return false, err - } - if !match { - continue - } - } - // kconfig rule - if rule.Kconfig != nil { - match, err := rule.Kconfig.Match() - if err != nil { - return false, err - } - if !match { - continue - } - } - return true, nil } return false, nil } diff --git a/source/custom/directory_features.go b/source/custom/directory_features.go new file mode 100644 index 000000000..333547a55 --- /dev/null +++ b/source/custom/directory_features.go @@ -0,0 +1,88 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package custom + +import ( + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + + "sigs.k8s.io/yaml" +) + +const Directory = "/etc/kubernetes/node-feature-discovery/custom.d" + +// getDirectoryFeatureConfig returns features configured in the "/etc/kubernetes/node-feature-discovery/custom.d" +// host directory and its 1st level subdirectories, which can be populated e.g. by ConfigMaps +func getDirectoryFeatureConfig() []FeatureSpec { + features := readDir(Directory, true) + //log.Printf("DEBUG: all configmap based custom feature specs: %+v", features) + return features +} + +func readDir(dirName string, recursive bool) []FeatureSpec { + features := make([]FeatureSpec, 0) + + log.Printf("DEBUG: getting files in %s", dirName) + files, err := ioutil.ReadDir(dirName) + if err != nil { + if os.IsNotExist(err) { + log.Printf("DEBUG: custom config directory %q does not exist", dirName) + } else { + log.Printf("ERROR: unable to access custom config directory %q, %v", dirName, err) + } + return features + } + + for _, file := range files { + fileName := filepath.Join(dirName, file.Name()) + + if file.IsDir() { + if recursive { + //log.Printf("DEBUG: going into dir %q", fileName) + features = append(features, readDir(fileName, false)...) + //} else { + // log.Printf("DEBUG: skipping dir %q", fileName) + } + continue + } + if strings.HasPrefix(file.Name(), ".") { + //log.Printf("DEBUG: skipping hidden file %q", fileName) + continue + } + //log.Printf("DEBUG: processing file %q", fileName) + + bytes, err := ioutil.ReadFile(fileName) + if err != nil { + log.Printf("ERROR: could not read custom config file %q, %v", fileName, err) + continue + } + //log.Printf("DEBUG: custom config rules raw: %s", string(bytes)) + + config := &[]FeatureSpec{} + err = yaml.UnmarshalStrict(bytes, config) + if err != nil { + log.Printf("ERROR: could not parse custom config file %q, %v", fileName, err) + continue + } + + features = append(features, *config...) + } + return features +} diff --git a/source/custom/rules/nodename_rule.go b/source/custom/rules/nodename_rule.go new file mode 100644 index 000000000..744f6dd6c --- /dev/null +++ b/source/custom/rules/nodename_rule.go @@ -0,0 +1,50 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package rules + +import ( + "log" + "os" + "regexp" +) + +var ( + nodeName = os.Getenv("NODE_NAME") +) + +// Rule that matches on nodenames configured in a ConfigMap +type NodenameRule []string + +// Force implementation of Rule +var _ Rule = NodenameRule{} + +func (n NodenameRule) Match() (bool, error) { + for _, nodenamePattern := range n { + log.Printf("DEBUG: matchNodename %s", nodenamePattern) + match, err := regexp.MatchString(nodenamePattern, nodeName) + if err != nil { + log.Printf("ERROR: nodename rule: invalid nodename regexp %q: %v", nodenamePattern, err) + continue + } + if !match { + //log.Printf("DEBUG: nodename rule: No match for pattern %q with node %q", nodenamePattern, nodeName) + continue + } + //log.Printf("DEBUG: nodename rule: Match for pattern %q with node %q", nodenamePattern, nodeName) + return true, nil + } + return false, nil +} diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 5526786d0..7880c0b48 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -21,12 +21,13 @@ import ( "flag" "fmt" "io/ioutil" + "path/filepath" "regexp" "strings" "time" - "github.com/onsi/ginkgo" - "github.com/onsi/gomega" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -40,6 +41,7 @@ import ( e2epod "k8s.io/kubernetes/test/e2e/framework/pod" master "sigs.k8s.io/node-feature-discovery/pkg/nfd-master" + "sigs.k8s.io/node-feature-discovery/source/custom" "sigs.k8s.io/yaml" ) @@ -90,18 +92,18 @@ func readConfig() { return } - ginkgo.By("Reading end-to-end test configuration file") + By("Reading end-to-end test configuration file") data, err := ioutil.ReadFile(*e2eConfigFile) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) - ginkgo.By("Parsing end-to-end test configuration data") + By("Parsing end-to-end test configuration data") err = yaml.Unmarshal(data, &conf) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) // Pre-compile node name matching regexps for name, nodeConf := range conf.DefaultFeatures.Nodes { nodeConf.nameRe, err = regexp.Compile(name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) conf.DefaultFeatures.Nodes[name] = nodeConf } } @@ -166,6 +168,13 @@ func createClusterRole(cs clientset.Interface) (*rbacv1.ClusterRole, error) { Resources: []string{"nodes"}, Verbs: []string{"get", "patch", "update"}, }, + { + // needed on OpenShift clusters + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + ResourceNames: []string{"hostaccess"}, + Verbs: []string{"use"}, + }, }, } return cs.RbacV1().ClusterRoles().Update(context.TODO(), cr, metav1.UpdateOptions{}) @@ -373,14 +382,14 @@ func newHostPathType(typ v1.HostPathType) *v1.HostPathType { // labels and annotations func cleanupNode(cs clientset.Interface) { nodeList, err := cs.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) for _, n := range nodeList.Items { var err error var node *v1.Node for retry := 0; retry < 5; retry++ { node, err = cs.CoreV1().Nodes().Get(context.TODO(), n.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) update := false // Remove labels @@ -403,7 +412,7 @@ func cleanupNode(cs clientset.Interface) { break } - ginkgo.By("Deleting NFD labels and annotations from node " + node.Name) + By("Deleting NFD labels and annotations from node " + node.Name) _, err = cs.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{}) if err != nil { time.Sleep(100 * time.Millisecond) @@ -412,7 +421,7 @@ func cleanupNode(cs clientset.Interface) { } } - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) } } @@ -420,42 +429,42 @@ func cleanupNode(cs clientset.Interface) { var _ = framework.KubeDescribe("[NFD] Node Feature Discovery", func() { f := framework.NewDefaultFramework("node-feature-discovery") - ginkgo.Context("when deploying a single nfd-master pod", func() { + Context("when deploying a single nfd-master pod", func() { var masterPod *v1.Pod - ginkgo.BeforeEach(func() { + BeforeEach(func() { err := configureRBAC(f.ClientSet, f.Namespace.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) // Launch nfd-master - ginkgo.By("Creating nfd master pod and nfd-master service") + By("Creating nfd master pod and nfd-master service") image := fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) masterPod = nfdMasterPod(image, false) masterPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), masterPod, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) // Create nfd-master service nfdSvc, err := createService(f.ClientSet, f.Namespace.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) - ginkgo.By("Waiting for the nfd-master pod to be running") - gomega.Expect(e2epod.WaitTimeoutForPodRunningInNamespace(f.ClientSet, masterPod.Name, masterPod.Namespace, time.Minute)).NotTo(gomega.HaveOccurred()) + By("Waiting for the nfd-master pod to be running") + Expect(e2epod.WaitTimeoutForPodRunningInNamespace(f.ClientSet, masterPod.Name, masterPod.Namespace, time.Minute)).NotTo(HaveOccurred()) - ginkgo.By("Waiting for the nfd-master service to be up") - gomega.Expect(e2enetwork.WaitForService(f.ClientSet, f.Namespace.Name, nfdSvc.ObjectMeta.Name, true, time.Second, 10*time.Second)).NotTo(gomega.HaveOccurred()) + By("Waiting for the nfd-master service to be up") + Expect(e2enetwork.WaitForService(f.ClientSet, f.Namespace.Name, nfdSvc.ObjectMeta.Name, true, time.Second, 10*time.Second)).NotTo(HaveOccurred()) }) - ginkgo.AfterEach(func() { + AfterEach(func() { err := deconfigureRBAC(f.ClientSet, f.Namespace.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) }) // // Simple test with only the fake source enabled // - ginkgo.Context("and a single worker pod with fake source enabled", func() { - ginkgo.It("it should decorate the node with the fake feature labels", func() { + Context("and a single worker pod with fake source enabled", func() { + It("it should decorate the node with the fake feature labels", func() { fakeFeatureLabels := map[string]string{ master.LabelNs + "/fake-fakefeature1": "true", @@ -467,34 +476,34 @@ var _ = framework.KubeDescribe("[NFD] Node Feature Discovery", func() { cleanupNode(f.ClientSet) // Launch nfd-worker - ginkgo.By("Creating a nfd worker pod") + By("Creating a nfd worker pod") image := fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) workerPod := nfdWorkerPod(image, []string{"--oneshot", "--sources=fake"}) workerPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), workerPod, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) - ginkgo.By("Waiting for the nfd-worker pod to succeed") - gomega.Expect(e2epod.WaitForPodSuccessInNamespace(f.ClientSet, workerPod.ObjectMeta.Name, f.Namespace.Name)).NotTo(gomega.HaveOccurred()) + By("Waiting for the nfd-worker pod to succeed") + Expect(e2epod.WaitForPodSuccessInNamespace(f.ClientSet, workerPod.ObjectMeta.Name, f.Namespace.Name)).NotTo(HaveOccurred()) workerPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), workerPod.ObjectMeta.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) - ginkgo.By(fmt.Sprintf("Making sure '%s' was decorated with the fake feature labels", workerPod.Spec.NodeName)) + By(fmt.Sprintf("Making sure '%s' was decorated with the fake feature labels", workerPod.Spec.NodeName)) node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), workerPod.Spec.NodeName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) for k, v := range fakeFeatureLabels { - gomega.Expect(node.Labels[k]).To(gomega.Equal(v)) + Expect(node.Labels[k]).To(Equal(v)) } // Check that there are no unexpected NFD labels for k := range node.Labels { if strings.HasPrefix(k, master.LabelNs) { - gomega.Expect(fakeFeatureLabels).Should(gomega.HaveKey(k)) + Expect(fakeFeatureLabels).Should(HaveKey(k)) } } - ginkgo.By("Deleting the node-feature-discovery worker pod") + By("Deleting the node-feature-discovery worker pod") err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), workerPod.ObjectMeta.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) cleanupNode(f.ClientSet) }) @@ -503,31 +512,31 @@ var _ = framework.KubeDescribe("[NFD] Node Feature Discovery", func() { // // More comprehensive test when --e2e-node-config is enabled // - ginkgo.Context("and nfd-workers as a daemonset with default sources enabled", func() { - ginkgo.It("the node labels and annotations listed in the e2e config should be present", func() { + Context("and nfd-workers as a daemonset with default sources enabled", func() { + It("the node labels and annotations listed in the e2e config should be present", func() { readConfig() if conf == nil { - ginkgo.Skip("no e2e-config was specified") + Skip("no e2e-config was specified") } if conf.DefaultFeatures == nil { - ginkgo.Skip("no 'defaultFeatures' specified in e2e-config") + Skip("no 'defaultFeatures' specified in e2e-config") } fConf := conf.DefaultFeatures // Remove pre-existing stale annotations and labels cleanupNode(f.ClientSet) - ginkgo.By("Creating nfd-worker daemonset") + By("Creating nfd-worker daemonset") workerDS := nfdWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{}) workerDS, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) - ginkgo.By("Waiting for daemonset pods to be ready") - gomega.Expect(e2epod.WaitForPodsReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(gomega.HaveOccurred()) + By("Waiting for daemonset pods to be ready") + Expect(e2epod.WaitForPodsReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) - ginkgo.By("Getting node objects") + By("Getting node objects") nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) for _, node := range nodeList.Items { var nodeConf *nodeConfig @@ -546,10 +555,10 @@ var _ = framework.KubeDescribe("[NFD] Node Feature Discovery", func() { // Check labels e2elog.Logf("verifying labels of node %q...", node.Name) for k, v := range nodeConf.ExpectedLabelValues { - gomega.Expect(node.Labels).To(gomega.HaveKeyWithValue(k, v)) + Expect(node.Labels).To(HaveKeyWithValue(k, v)) } for k := range nodeConf.ExpectedLabelKeys { - gomega.Expect(node.Labels).To(gomega.HaveKey(k)) + Expect(node.Labels).To(HaveKey(k)) } for k := range node.Labels { if strings.HasPrefix(k, master.LabelNs) { @@ -560,17 +569,17 @@ var _ = framework.KubeDescribe("[NFD] Node Feature Discovery", func() { continue } // Ignore if the label key was not whitelisted - gomega.Expect(fConf.LabelWhitelist).NotTo(gomega.HaveKey(k)) + Expect(fConf.LabelWhitelist).NotTo(HaveKey(k)) } } // Check annotations e2elog.Logf("verifying annotations of node %q...", node.Name) for k, v := range nodeConf.ExpectedAnnotationValues { - gomega.Expect(node.Annotations).To(gomega.HaveKeyWithValue(k, v)) + Expect(node.Annotations).To(HaveKeyWithValue(k, v)) } for k := range nodeConf.ExpectedAnnotationKeys { - gomega.Expect(node.Annotations).To(gomega.HaveKey(k)) + Expect(node.Annotations).To(HaveKey(k)) } for k := range node.Annotations { if strings.HasPrefix(k, master.AnnotationNsBase) { @@ -581,23 +590,181 @@ var _ = framework.KubeDescribe("[NFD] Node Feature Discovery", func() { continue } // Ignore if the annotation was not whitelisted - gomega.Expect(fConf.AnnotationWhitelist).NotTo(gomega.HaveKey(k)) + Expect(fConf.AnnotationWhitelist).NotTo(HaveKey(k)) } } // Node running nfd-master should have master version annotation if node.Name == masterPod.Spec.NodeName { - gomega.Expect(node.Annotations).To(gomega.HaveKey(master.AnnotationNsBase + "master.version")) + Expect(node.Annotations).To(HaveKey(master.AnnotationNsBase + "master.version")) } } - ginkgo.By("Deleting nfd-worker daemonset") + By("Deleting nfd-worker daemonset") err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(context.TODO(), workerDS.ObjectMeta.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) cleanupNode(f.ClientSet) }) }) + + // + // Test custom nodename source configured in 2 additional ConfigMaps + // + Context("and nfd-workers as a daemonset with 2 additional configmaps for the custom source configured", func() { + It("the nodename matching features listed in the configmaps should be present", func() { + // Remove pre-existing stale annotations and labels + cleanupNode(f.ClientSet) + + By("Getting a worker node") + + // We need a valid nodename for the configmap + nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeList.Items)).ToNot(BeZero()) + + targetNodeName := "" + for _, node := range nodeList.Items { + if _, ok := node.Labels["node-role.kubernetes.io/master"]; !ok { + targetNodeName = node.Name + break + } + } + Expect(targetNodeName).ToNot(BeEmpty(), "No worker node found") + + // create a wildcard name as well for this node + targetNodeNameWildcard := fmt.Sprintf("%s.*%s", targetNodeName[:2], targetNodeName[4:]) + + By("Creating the configmaps") + targetLabelName := "nodename-test" + targetLabelValue := "true" + + targetLabelNameWildcard := "nodename-test-wildcard" + targetLabelValueWildcard := "customValue" + + targetLabelNameNegative := "nodename-test-negative" + + // create 2 configmaps + data1 := make(map[string]string) + data1["custom1.conf"] = ` +- name: ` + targetLabelName + ` + matchOn: + # default value is true + - nodename: + - ` + targetNodeName + + cm1 := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-config-extra-" + string(uuid.NewUUID()), + }, + Data: data1, + } + cm1, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm1, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + data2 := make(map[string]string) + data2["custom1.conf"] = ` +- name: ` + targetLabelNameWildcard + ` + value: ` + targetLabelValueWildcard + ` + matchOn: + - nodename: + - ` + targetNodeNameWildcard + ` +- name: ` + targetLabelNameNegative + ` + matchOn: + - nodename: + - "thisNameShouldNeverMatch"` + + cm2 := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-config-extra-" + string(uuid.NewUUID()), + }, + Data: data2, + } + cm2, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm2, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Creating nfd-worker daemonset with configmap mounted") + workerDS := nfdWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{}) + + // add configmap mount config + volumeName1 := "custom-configs-extra1" + volumeName2 := "custom-configs-extra2" + workerDS.Spec.Template.Spec.Volumes = append(workerDS.Spec.Template.Spec.Volumes, + v1.Volume{ + Name: volumeName1, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: cm1.Name, + }, + }, + }, + }, + v1.Volume{ + Name: volumeName2, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: cm2.Name, + }, + }, + }, + }, + ) + workerDS.Spec.Template.Spec.Containers[0].VolumeMounts = append(workerDS.Spec.Template.Spec.Containers[0].VolumeMounts, + v1.VolumeMount{ + Name: volumeName1, + ReadOnly: true, + MountPath: filepath.Join(custom.Directory, "cm1"), + }, + v1.VolumeMount{ + Name: volumeName2, + ReadOnly: true, + MountPath: filepath.Join(custom.Directory, "cm2"), + }, + ) + + workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for daemonset pods to be ready") + Expect(e2epod.WaitForPodsReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) + + By("Getting target node and checking labels") + targetNode, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), targetNodeName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + + labelFound := false + labelWildcardFound := false + labelNegativeFound := false + for k := range targetNode.Labels { + if strings.Contains(k, targetLabelName) { + if targetNode.Labels[k] == targetLabelValue { + labelFound = true + } + } + if strings.Contains(k, targetLabelNameWildcard) { + if targetNode.Labels[k] == targetLabelValueWildcard { + labelWildcardFound = true + } + } + if strings.Contains(k, targetLabelNameNegative) { + labelNegativeFound = true + } + } + + Expect(labelFound).To(BeTrue(), "label not found!") + Expect(labelWildcardFound).To(BeTrue(), "label for wildcard nodename not found!") + Expect(labelNegativeFound).To(BeFalse(), "label for not existing nodename found!") + + By("Deleting nfd-worker daemonset") + err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(context.TODO(), workerDS.ObjectMeta.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + cleanupNode(f.ClientSet) + }) + }) + }) })