mirrors/node-feature-discovery
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/node-feature-discovery.git synced 2024-12-14 11:57:51 +00:00
Code Activity

source/custom: Add nodename rule

Browse source 
There are cases when the only available metadata for discovering
features is the node's name. The "nodename" rule extends the custom
source and matches when the node's name matches one of the given
nodename regexp patterns.
It is also possible now to set an optional "value" on custom rules,
which overrides the default "true" label value in case the rule matches.
In order to allow more dynamic configurations without having to modify
the complete worker configuration, custom rules are additionally read
from a "custom.d" directory now. Typically that directory will be filled
by mounting one or more ConfigMaps.

Signed-off-by: Marc Sluiter <msluiter@redhat.com>
This commit is contained in:
Marc Sluiter 2021-01-21 00:48:40 +01:00
parent 3e00bfb2ae
commit 7038e49d02
No known key found for this signature in database
GPG key ID: 17E52E106BC9561E
10 changed files with 534 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
deployment/node-feature-discovery/values.yaml
View file

@ -161,6 +161,10 @@ worker:
# vendor: ["15b3"]
# device: ["1014", "1017"]
# loadedKMod : ["vendor_kmod1", "vendor_kmod2"]
# - name: "feature.by.nodename"
# value: customValue
# matchOn:
# - nodename: ["worker-0", "my-.*-node"]
### <NFD-WORKER-CONF-END-DO-NOT-REMOVE>
podSecurityContext: {}

40
docs/get-started/features.md
View file

@ -134,6 +134,17 @@ examples how to set-up and manage the worker configuration.
To aid in making Custom Features clearer, we define a general and a per rule
nomenclature, keeping things as consistent as possible.
#### Additional configuration directory
Additionally to the rules defined in the nfd-worker configuration file, the
Custom feature can read more configuration files located in the
`/etc/kubernetes/node-feature-discovery/custom.d/` directory. This makes more
dynamic and flexible configuration easier. This directory must be available
inside the NFD worker container, so Volumes and VolumeMounts must be used for
mounting e.g. ConfigMap(s). The example deployment manifests provide an example
(commented out) for providing Custom configuration with an additional
ConfigMap, mounted into the `custom.d` directory.
#### General Nomenclature & Definitions
```
@ -151,6 +162,7 @@ file.
sources:
custom:
- name: <feature name>
value: <optional feature value, defaults to "true">
matchOn:
- <Rule-1>: <Rule-1 Input>
[<Rule-2>: <Rule-2 Input>]
@ -291,6 +303,26 @@ Matching is done by performing logical _AND_ for each provided Element, i.e the
Rule will match if all provided Elements (kernel config options) are enabled
(`y` or `m`) or matching `=<value>` in the kernel.
##### Nodename Rule
###### Nomenclature
```
Element :A nodename regexp pattern
```
The Rule allows matching the node's name against a provided list of Elements.
###### Format
```yaml
nodename: [ <nodename regexp pattern>, ... ]
```
Matching is done by performing logical _OR_ for each provided Element, i.e the
Rule will match if one of the provided Elements (nodename regexp pattern)
matches the node's name.
#### Example
```yaml
@ -328,6 +360,10 @@ custom:
matchOn:
- kConfig: ["GCC_VERSION=100101"]
loadedKMod: ["kmod1"]
- name: "my.datacenter"
value: "datacenter-1"
matchOn:
- nodename: [ "node-datacenter1-rack.*-server.*" ]
```
__In the example above:__
@ -360,6 +396,10 @@ __In the example above:__
`feature.node.kubernetes.io/custom-my.kernel.modulecompiler=true` if the
in-tree `kmod1` kernel module is loaded __AND__ it's built with
`GCC_VERSION=100101`.
- A node would contain the label:
`feature.node.kubernetes.io/my.datacenter=datacenter-1` if the node's name
matches the `node-datacenter1-rack.*-server.*` pattern, e.g.
`node-datacenter1-rack2-server42`
#### Statically defined features

30
nfd-daemonset-combined.yaml.template
View file

@ -115,6 +115,11 @@ spec:
- name: nfd-worker-conf
mountPath: "/etc/kubernetes/node-feature-discovery"
readOnly: true
## Example for more custom configs in an additional configmap (1/3)
## Mounting into subdirectories of custom.d makes it easy to use multiple configmaps
# - name: custom-source-extra-rules
# mountPath: "/etc/kubernetes/node-feature-discovery/custom.d/extra-rules-1"
# readOnly: true
volumes:
- name: host-boot
hostPath:
@ -134,6 +139,10 @@ spec:
- name: nfd-worker-conf
configMap:
name: nfd-worker-conf
## Example for more custom configs in an additional configmap (2/3)
# - name: custom-source-extra-rules
# configMap:
# name: custom-source-extra-rules
---
apiVersion: v1
kind: ConfigMap
@ -232,4 +241,25 @@ data:
# vendor: ["15b3"]
# device: ["1014", "1017"]
# loadedKMod : ["vendor_kmod1", "vendor_kmod2"]
# - name: "feature.by.nodename"
# value: customValue
# matchOn:
# - nodename: ["worker-0", "my-.*-node"]
### <NFD-WORKER-CONF-END-DO-NOT-REMOVE>
---
## Example for more custom configs in an additional configmap (3/3)
#apiVersion: v1
#kind: ConfigMap
#metadata:
# name: custom-source-extra-rules
# namespace: node-feature-discovery
#data:
## Filename doesn't matter, and there can be multiple. They just need to be unique.
# custom.conf: |
# - name: "more.kernel.features"
# matchOn:
# - loadedKMod: ["example_kmod3"]
# - name: "more.features.by.nodename"
# value: customValue
# matchOn:
# - nodename: ["special-.*-node-.*"]

30
nfd-worker-daemonset.yaml.template
View file

@ -68,6 +68,11 @@ spec:
- name: nfd-worker-conf
mountPath: "/etc/kubernetes/node-feature-discovery"
readOnly: true
## Example for more custom configs in an additional configmap (1/3)
## Mounting into subdirectories of custom.d makes it easy to use multiple configmaps
# - name: custom-source-extra-rules
# mountPath: "/etc/kubernetes/node-feature-discovery/custom.d/extra-rules-1"
# readOnly: true
## Enable TLS authentication (2/3)
# - name: nfd-ca-cert
# mountPath: "/etc/kubernetes/node-feature-discovery/trust"
@ -94,6 +99,10 @@ spec:
- name: nfd-worker-conf
configMap:
name: nfd-worker-conf
## Example for more custom configs in an additional configmap (2/3)
# - name: custom-source-extra-rules
# configMap:
# name: custom-source-extra-rules
## Enable TLS authentication (3/3)
# - name: nfd-ca-cert
# configMap:
@ -199,4 +208,25 @@ data:
# vendor: ["15b3"]
# device: ["1014", "1017"]
# loadedKMod : ["vendor_kmod1", "vendor_kmod2"]
# - name: "feature.by.nodename"
# value: customValue
# matchOn:
# - nodename: ["worker-0", "my-.*-node"]
### <NFD-WORKER-CONF-END-DO-NOT-REMOVE>
---
## Example for more custom configs in an additional configmap (3/3)
#apiVersion: v1
#kind: ConfigMap
#metadata:
# name: custom-source-extra-rules
# namespace: node-feature-discovery
#data:
## Filename doesn't matter, and there can be multiple. They just need to be unique.
# custom.conf: |
# - name: "more.kernel.features"
# matchOn:
# - loadedKMod: ["example_kmod3"]
# - name: "more.features.by.nodename"
# value: customValue
# matchOn:
# - nodename: ["special-.*-node-.*"]

30
nfd-worker-job.yaml.template
View file

@ -72,6 +72,11 @@ spec:
- name: nfd-worker-conf
mountPath: "/etc/kubernetes/node-feature-discovery"
readOnly: true
## Example for more custom configs in an additional configmap (1/3)
## Mounting into subdirectories of custom.d makes it easy to use multiple configmaps
# - name: custom-source-extra-rules
# mountPath: "/etc/kubernetes/node-feature-discovery/custom.d/extra-rules-1"
# readOnly: true
## Enable TLS authentication (2/3)
# - name: nfd-ca-cert
# mountPath: "/etc/kubernetes/node-feature-discovery/trust"
@ -99,6 +104,10 @@ spec:
- name: nfd-worker-conf
configMap:
name: nfd-worker-conf
## Example for more custom configs in an additional configmap (2/3)
# - name: custom-source-extra-rules
# configMap:
# name: custom-source-extra-rules
## Enable TLS authentication (3/3)
# - name: nfd-ca-cert
# configMap:
@ -204,4 +213,25 @@ data:
# vendor: ["15b3"]
# device: ["1014", "1017"]
# loadedKMod : ["vendor_kmod1", "vendor_kmod2"]
# - name: "feature.by.nodename"
# value: customValue
# matchOn:
# - nodename: ["worker-0", "my-.*-node"]
### <NFD-WORKER-CONF-END-DO-NOT-REMOVE>
---
## Example for more custom configs in an additional configmap (3/3)
#apiVersion: v1
#kind: ConfigMap
#metadata:
# name: custom-source-extra-rules
# namespace: node-feature-discovery
#data:
## Filename doesn't matter, and there can be multiple. They just need to be unique.
# custom.conf: |
# - name: "more.kernel.features"
# matchOn:
# - loadedKMod: ["example_kmod3"]
# - name: "more.features.by.nodename"
# value: customValue
# matchOn:
# - nodename: ["special-.*-node-.*"]

4
nfd-worker.conf.example
View file

@ -88,3 +88,7 @@
# vendor: ["15b3"]
# device: ["1014", "1017"]
# loadedKMod : ["vendor_kmod1", "vendor_kmod2"]
# - name: "feature.by.nodename"
# value: customValue
# matchOn:
# - nodename: ["worker-0", "my-.*-node"]

85
source/custom/custom.go
View file

@ -18,6 +18,7 @@ package custom
import (
"log"
"reflect"
"sigs.k8s.io/node-feature-discovery/source"
"sigs.k8s.io/node-feature-discovery/source/custom/rules"
@ -30,10 +31,12 @@ type MatchRule struct {
LoadedKMod *rules.LoadedKModRule `json:"loadedKMod,omitempty"`
CpuID *rules.CpuIDRule `json:"cpuId,omitempty"`
Kconfig *rules.KconfigRule `json:"kConfig,omitempty"`
Nodename *rules.NodenameRule `json:"nodename,omitempty"`
}
type FeatureSpec struct {
Name string `json:"name"`
Value *string `json:"value"`
MatchOn []MatchRule `json:"matchOn"`
}
@ -72,6 +75,7 @@ func (s *Source) SetConfig(conf source.Config) {
func (s Source) Discover() (source.Features, error) {
features := source.Features{}
allFeatureConfig := append(getStaticFeatureConfig(), *s.config...)
allFeatureConfig = append(allFeatureConfig, getDirectoryFeatureConfig()...)
log.Printf("INFO: Custom features: %+v", allFeatureConfig)
// Iterate over features
for _, customFeature := range allFeatureConfig {
@ -81,7 +85,11 @@ func (s Source) Discover() (source.Features, error) {
continue
}
if featureExist {
features[customFeature.Name] = true
var value interface{} = true
if customFeature.Value != nil {
value = *customFeature.Value
}
features[customFeature.Name] = value
}
}
return features, nil
@ -90,58 +98,37 @@ func (s Source) Discover() (source.Features, error) {
// Process a single feature by Matching on the defined rules.
// A feature is present if all defined Rules in a MatchRule return a match.
func (s Source) discoverFeature(feature FeatureSpec) (bool, error) {
for _, rule := range feature.MatchOn {
// PCI ID rule
if rule.PciID != nil {
match, err := rule.PciID.Match()
if err != nil {
return false, err
}
if !match {
continue
}
for _, matchRules := range feature.MatchOn {
allRules := []rules.Rule{
matchRules.PciID,
matchRules.UsbID,
matchRules.LoadedKMod,
matchRules.CpuID,
matchRules.Kconfig,
matchRules.Nodename,
}
// USB ID rule
if rule.UsbID != nil {
match, err := rule.UsbID.Match()
if err != nil {
return false, err
}
if !match {
continue
// return true, nil if all rules match
matchRules := func(rules []rules.Rule) (bool, error) {
for _, rule := range rules {
if reflect.ValueOf(rule).IsNil() {
continue
}
if match, err := rule.Match(); err != nil {
return false, err
} else if !match {
return false, nil
}
}
return true, nil
}
// Loaded kernel module rule
if rule.LoadedKMod != nil {
match, err := rule.LoadedKMod.Match()
if err != nil {
return false, err
}
if !match {
continue
}
if match, err := matchRules(allRules); err != nil {
return false, err
} else if match {
return true, nil
}
// cpuid rule
if rule.CpuID != nil {
match, err := rule.CpuID.Match()
if err != nil {
return false, err
}
if !match {
continue
}
}
// kconfig rule
if rule.Kconfig != nil {
match, err := rule.Kconfig.Match()
if err != nil {
return false, err
}
if !match {
continue
}
}
return true, nil
}
return false, nil
}

88
source/custom/directory_features.go Normal file
View file

@ -0,0 +1,88 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package custom
import (
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"
"sigs.k8s.io/yaml"
)
const Directory = "/etc/kubernetes/node-feature-discovery/custom.d"
// getDirectoryFeatureConfig returns features configured in the "/etc/kubernetes/node-feature-discovery/custom.d"
// host directory and its 1st level subdirectories, which can be populated e.g. by ConfigMaps
func getDirectoryFeatureConfig() []FeatureSpec {
features := readDir(Directory, true)
//log.Printf("DEBUG: all configmap based custom feature specs: %+v", features)
return features
}
func readDir(dirName string, recursive bool) []FeatureSpec {
features := make([]FeatureSpec, 0)
log.Printf("DEBUG: getting files in %s", dirName)
files, err := ioutil.ReadDir(dirName)
if err != nil {
if os.IsNotExist(err) {
log.Printf("DEBUG: custom config directory %q does not exist", dirName)
} else {
log.Printf("ERROR: unable to access custom config directory %q, %v", dirName, err)
}
return features
}
for _, file := range files {
fileName := filepath.Join(dirName, file.Name())
if file.IsDir() {
if recursive {
//log.Printf("DEBUG: going into dir %q", fileName)
features = append(features, readDir(fileName, false)...)
//} else {
// log.Printf("DEBUG: skipping dir %q", fileName)
}
continue
}
if strings.HasPrefix(file.Name(), ".") {
//log.Printf("DEBUG: skipping hidden file %q", fileName)
continue
}
//log.Printf("DEBUG: processing file %q", fileName)
bytes, err := ioutil.ReadFile(fileName)
if err != nil {
log.Printf("ERROR: could not read custom config file %q, %v", fileName, err)
continue
}
//log.Printf("DEBUG: custom config rules raw: %s", string(bytes))
config := &[]FeatureSpec{}
err = yaml.UnmarshalStrict(bytes, config)
if err != nil {
log.Printf("ERROR: could not parse custom config file %q, %v", fileName, err)
continue
}
features = append(features, *config...)
}
return features
}

50
source/custom/rules/nodename_rule.go Normal file
View file

@ -0,0 +1,50 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rules
import (
"log"
"os"
"regexp"
)
var (
nodeName = os.Getenv("NODE_NAME")
)
// Rule that matches on nodenames configured in a ConfigMap
type NodenameRule []string
// Force implementation of Rule
var _ Rule = NodenameRule{}
func (n NodenameRule) Match() (bool, error) {
for _, nodenamePattern := range n {
log.Printf("DEBUG: matchNodename %s", nodenamePattern)
match, err := regexp.MatchString(nodenamePattern, nodeName)
if err != nil {
log.Printf("ERROR: nodename rule: invalid nodename regexp %q: %v", nodenamePattern, err)
continue
}
if !match {
//log.Printf("DEBUG: nodename rule: No match for pattern %q with node %q", nodenamePattern, nodeName)
continue
}
//log.Printf("DEBUG: nodename rule: Match for pattern %q with node %q", nodenamePattern, nodeName)
return true, nil
}
return false, nil
}

277
test/e2e/node_feature_discovery.go
View file

@ -21,12 +21,13 @@ import (
"flag"
"fmt"
"io/ioutil"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
@ -40,6 +41,7 @@ import (
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
master "sigs.k8s.io/node-feature-discovery/pkg/nfd-master"
"sigs.k8s.io/node-feature-discovery/source/custom"
"sigs.k8s.io/yaml"
)
@ -90,18 +92,18 @@ func readConfig() {
return
}
ginkgo.By("Reading end-to-end test configuration file")
By("Reading end-to-end test configuration file")
data, err := ioutil.ReadFile(*e2eConfigFile)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
ginkgo.By("Parsing end-to-end test configuration data")
By("Parsing end-to-end test configuration data")
err = yaml.Unmarshal(data, &conf)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
// Pre-compile node name matching regexps
for name, nodeConf := range conf.DefaultFeatures.Nodes {
nodeConf.nameRe, err = regexp.Compile(name)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
conf.DefaultFeatures.Nodes[name] = nodeConf
}
}
@ -166,6 +168,13 @@ func createClusterRole(cs clientset.Interface) (*rbacv1.ClusterRole, error) {
Resources: []string{"nodes"},
Verbs: []string{"get", "patch", "update"},
},
{
// needed on OpenShift clusters
APIGroups: []string{"security.openshift.io"},
Resources: []string{"securitycontextconstraints"},
ResourceNames: []string{"hostaccess"},
Verbs: []string{"use"},
},
},
}
return cs.RbacV1().ClusterRoles().Update(context.TODO(), cr, metav1.UpdateOptions{})
@ -373,14 +382,14 @@ func newHostPathType(typ v1.HostPathType) *v1.HostPathType {
// labels and annotations
func cleanupNode(cs clientset.Interface) {
nodeList, err := cs.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
for _, n := range nodeList.Items {
var err error
var node *v1.Node
for retry := 0; retry < 5; retry++ {
node, err = cs.CoreV1().Nodes().Get(context.TODO(), n.Name, metav1.GetOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
update := false
// Remove labels
@ -403,7 +412,7 @@ func cleanupNode(cs clientset.Interface) {
break
}
ginkgo.By("Deleting NFD labels and annotations from node " + node.Name)
By("Deleting NFD labels and annotations from node " + node.Name)
_, err = cs.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{})
if err != nil {
time.Sleep(100 * time.Millisecond)
@ -412,7 +421,7 @@ func cleanupNode(cs clientset.Interface) {
}
}
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
}
}
@ -420,42 +429,42 @@ func cleanupNode(cs clientset.Interface) {
var _ = framework.KubeDescribe("[NFD] Node Feature Discovery", func() {
f := framework.NewDefaultFramework("node-feature-discovery")
ginkgo.Context("when deploying a single nfd-master pod", func() {
Context("when deploying a single nfd-master pod", func() {
var masterPod *v1.Pod
ginkgo.BeforeEach(func() {
BeforeEach(func() {
err := configureRBAC(f.ClientSet, f.Namespace.Name)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
// Launch nfd-master
ginkgo.By("Creating nfd master pod and nfd-master service")
By("Creating nfd master pod and nfd-master service")
image := fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)
masterPod = nfdMasterPod(image, false)
masterPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), masterPod, metav1.CreateOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
// Create nfd-master service
nfdSvc, err := createService(f.ClientSet, f.Namespace.Name)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
ginkgo.By("Waiting for the nfd-master pod to be running")
gomega.Expect(e2epod.WaitTimeoutForPodRunningInNamespace(f.ClientSet, masterPod.Name, masterPod.Namespace, time.Minute)).NotTo(gomega.HaveOccurred())
By("Waiting for the nfd-master pod to be running")
Expect(e2epod.WaitTimeoutForPodRunningInNamespace(f.ClientSet, masterPod.Name, masterPod.Namespace, time.Minute)).NotTo(HaveOccurred())
ginkgo.By("Waiting for the nfd-master service to be up")
gomega.Expect(e2enetwork.WaitForService(f.ClientSet, f.Namespace.Name, nfdSvc.ObjectMeta.Name, true, time.Second, 10*time.Second)).NotTo(gomega.HaveOccurred())
By("Waiting for the nfd-master service to be up")
Expect(e2enetwork.WaitForService(f.ClientSet, f.Namespace.Name, nfdSvc.ObjectMeta.Name, true, time.Second, 10*time.Second)).NotTo(HaveOccurred())
})
ginkgo.AfterEach(func() {
AfterEach(func() {
err := deconfigureRBAC(f.ClientSet, f.Namespace.Name)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
})
//
// Simple test with only the fake source enabled
//
ginkgo.Context("and a single worker pod with fake source enabled", func() {
ginkgo.It("it should decorate the node with the fake feature labels", func() {
Context("and a single worker pod with fake source enabled", func() {
It("it should decorate the node with the fake feature labels", func() {
fakeFeatureLabels := map[string]string{
master.LabelNs + "/fake-fakefeature1": "true",
@ -467,34 +476,34 @@ var _ = framework.KubeDescribe("[NFD] Node Feature Discovery", func() {
cleanupNode(f.ClientSet)
// Launch nfd-worker
ginkgo.By("Creating a nfd worker pod")
By("Creating a nfd worker pod")
image := fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)
workerPod := nfdWorkerPod(image, []string{"--oneshot", "--sources=fake"})
workerPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), workerPod, metav1.CreateOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
ginkgo.By("Waiting for the nfd-worker pod to succeed")
gomega.Expect(e2epod.WaitForPodSuccessInNamespace(f.ClientSet, workerPod.ObjectMeta.Name, f.Namespace.Name)).NotTo(gomega.HaveOccurred())
By("Waiting for the nfd-worker pod to succeed")
Expect(e2epod.WaitForPodSuccessInNamespace(f.ClientSet, workerPod.ObjectMeta.Name, f.Namespace.Name)).NotTo(HaveOccurred())
workerPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), workerPod.ObjectMeta.Name, metav1.GetOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
ginkgo.By(fmt.Sprintf("Making sure '%s' was decorated with the fake feature labels", workerPod.Spec.NodeName))
By(fmt.Sprintf("Making sure '%s' was decorated with the fake feature labels", workerPod.Spec.NodeName))
node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), workerPod.Spec.NodeName, metav1.GetOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
for k, v := range fakeFeatureLabels {
gomega.Expect(node.Labels[k]).To(gomega.Equal(v))
Expect(node.Labels[k]).To(Equal(v))
}
// Check that there are no unexpected NFD labels
for k := range node.Labels {
if strings.HasPrefix(k, master.LabelNs) {
gomega.Expect(fakeFeatureLabels).Should(gomega.HaveKey(k))
Expect(fakeFeatureLabels).Should(HaveKey(k))
}
}
ginkgo.By("Deleting the node-feature-discovery worker pod")
By("Deleting the node-feature-discovery worker pod")
err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), workerPod.ObjectMeta.Name, metav1.DeleteOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
cleanupNode(f.ClientSet)
})
@ -503,31 +512,31 @@ var _ = framework.KubeDescribe("[NFD] Node Feature Discovery", func() {
//
// More comprehensive test when --e2e-node-config is enabled
//
ginkgo.Context("and nfd-workers as a daemonset with default sources enabled", func() {
ginkgo.It("the node labels and annotations listed in the e2e config should be present", func() {
Context("and nfd-workers as a daemonset with default sources enabled", func() {
It("the node labels and annotations listed in the e2e config should be present", func() {
readConfig()
if conf == nil {
ginkgo.Skip("no e2e-config was specified")
Skip("no e2e-config was specified")
}
if conf.DefaultFeatures == nil {
ginkgo.Skip("no 'defaultFeatures' specified in e2e-config")
Skip("no 'defaultFeatures' specified in e2e-config")
}
fConf := conf.DefaultFeatures
// Remove pre-existing stale annotations and labels
cleanupNode(f.ClientSet)
ginkgo.By("Creating nfd-worker daemonset")
By("Creating nfd-worker daemonset")
workerDS := nfdWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{})
workerDS, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
ginkgo.By("Waiting for daemonset pods to be ready")
gomega.Expect(e2epod.WaitForPodsReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(gomega.HaveOccurred())
By("Waiting for daemonset pods to be ready")
Expect(e2epod.WaitForPodsReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred())
ginkgo.By("Getting node objects")
By("Getting node objects")
nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
for _, node := range nodeList.Items {
var nodeConf *nodeConfig
@ -546,10 +555,10 @@ var _ = framework.KubeDescribe("[NFD] Node Feature Discovery", func() {
// Check labels
e2elog.Logf("verifying labels of node %q...", node.Name)
for k, v := range nodeConf.ExpectedLabelValues {
gomega.Expect(node.Labels).To(gomega.HaveKeyWithValue(k, v))
Expect(node.Labels).To(HaveKeyWithValue(k, v))
}
for k := range nodeConf.ExpectedLabelKeys {
gomega.Expect(node.Labels).To(gomega.HaveKey(k))
Expect(node.Labels).To(HaveKey(k))
}
for k := range node.Labels {
if strings.HasPrefix(k, master.LabelNs) {
@ -560,17 +569,17 @@ var _ = framework.KubeDescribe("[NFD] Node Feature Discovery", func() {
continue
}
// Ignore if the label key was not whitelisted
gomega.Expect(fConf.LabelWhitelist).NotTo(gomega.HaveKey(k))
Expect(fConf.LabelWhitelist).NotTo(HaveKey(k))
}
}
// Check annotations
e2elog.Logf("verifying annotations of node %q...", node.Name)
for k, v := range nodeConf.ExpectedAnnotationValues {
gomega.Expect(node.Annotations).To(gomega.HaveKeyWithValue(k, v))
Expect(node.Annotations).To(HaveKeyWithValue(k, v))
}
for k := range nodeConf.ExpectedAnnotationKeys {
gomega.Expect(node.Annotations).To(gomega.HaveKey(k))
Expect(node.Annotations).To(HaveKey(k))
}
for k := range node.Annotations {
if strings.HasPrefix(k, master.AnnotationNsBase) {
@ -581,23 +590,181 @@ var _ = framework.KubeDescribe("[NFD] Node Feature Discovery", func() {
continue
}
// Ignore if the annotation was not whitelisted
gomega.Expect(fConf.AnnotationWhitelist).NotTo(gomega.HaveKey(k))
Expect(fConf.AnnotationWhitelist).NotTo(HaveKey(k))
}
}
// Node running nfd-master should have master version annotation
if node.Name == masterPod.Spec.NodeName {
gomega.Expect(node.Annotations).To(gomega.HaveKey(master.AnnotationNsBase + "master.version"))
Expect(node.Annotations).To(HaveKey(master.AnnotationNsBase + "master.version"))
}
}
ginkgo.By("Deleting nfd-worker daemonset")
By("Deleting nfd-worker daemonset")
err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(context.TODO(), workerDS.ObjectMeta.Name, metav1.DeleteOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expect(err).NotTo(HaveOccurred())
cleanupNode(f.ClientSet)
})
})
//
// Test custom nodename source configured in 2 additional ConfigMaps
//
Context("and nfd-workers as a daemonset with 2 additional configmaps for the custom source configured", func() {
It("the nodename matching features listed in the configmaps should be present", func() {
// Remove pre-existing stale annotations and labels
cleanupNode(f.ClientSet)
By("Getting a worker node")
// We need a valid nodename for the configmap
nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeList.Items)).ToNot(BeZero())
targetNodeName := ""
for _, node := range nodeList.Items {
if _, ok := node.Labels["node-role.kubernetes.io/master"]; !ok {
targetNodeName = node.Name
break
}
}
Expect(targetNodeName).ToNot(BeEmpty(), "No worker node found")
// create a wildcard name as well for this node
targetNodeNameWildcard := fmt.Sprintf("%s.*%s", targetNodeName[:2], targetNodeName[4:])
By("Creating the configmaps")
targetLabelName := "nodename-test"
targetLabelValue := "true"
targetLabelNameWildcard := "nodename-test-wildcard"
targetLabelValueWildcard := "customValue"
targetLabelNameNegative := "nodename-test-negative"
// create 2 configmaps
data1 := make(map[string]string)
data1["custom1.conf"] = `
- name: ` + targetLabelName + `
matchOn:
# default value is true
- nodename:
- ` + targetNodeName
cm1 := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "custom-config-extra-" + string(uuid.NewUUID()),
},
Data: data1,
}
cm1, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm1, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
data2 := make(map[string]string)
data2["custom1.conf"] = `
- name: ` + targetLabelNameWildcard + `
value: ` + targetLabelValueWildcard + `
matchOn:
- nodename:
- ` + targetNodeNameWildcard + `
- name: ` + targetLabelNameNegative + `
matchOn:
- nodename:
- "thisNameShouldNeverMatch"`
cm2 := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "custom-config-extra-" + string(uuid.NewUUID()),
},
Data: data2,
}
cm2, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm2, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
By("Creating nfd-worker daemonset with configmap mounted")
workerDS := nfdWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{})
// add configmap mount config
volumeName1 := "custom-configs-extra1"
volumeName2 := "custom-configs-extra2"
workerDS.Spec.Template.Spec.Volumes = append(workerDS.Spec.Template.Spec.Volumes,
v1.Volume{
Name: volumeName1,
VolumeSource: v1.VolumeSource{
ConfigMap: &v1.ConfigMapVolumeSource{
LocalObjectReference: v1.LocalObjectReference{
Name: cm1.Name,
},
},
},
},
v1.Volume{
Name: volumeName2,
VolumeSource: v1.VolumeSource{
ConfigMap: &v1.ConfigMapVolumeSource{
LocalObjectReference: v1.LocalObjectReference{
Name: cm2.Name,
},
},
},
},
)
workerDS.Spec.Template.Spec.Containers[0].VolumeMounts = append(workerDS.Spec.Template.Spec.Containers[0].VolumeMounts,
v1.VolumeMount{
Name: volumeName1,
ReadOnly: true,
MountPath: filepath.Join(custom.Directory, "cm1"),
},
v1.VolumeMount{
Name: volumeName2,
ReadOnly: true,
MountPath: filepath.Join(custom.Directory, "cm2"),
},
)
workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
By("Waiting for daemonset pods to be ready")
Expect(e2epod.WaitForPodsReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred())
By("Getting target node and checking labels")
targetNode, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), targetNodeName, metav1.GetOptions{})
Expect(err).ToNot(HaveOccurred())
labelFound := false
labelWildcardFound := false
labelNegativeFound := false
for k := range targetNode.Labels {
if strings.Contains(k, targetLabelName) {
if targetNode.Labels[k] == targetLabelValue {
labelFound = true
}
}
if strings.Contains(k, targetLabelNameWildcard) {
if targetNode.Labels[k] == targetLabelValueWildcard {
labelWildcardFound = true
}
}
if strings.Contains(k, targetLabelNameNegative) {
labelNegativeFound = true
}
}
Expect(labelFound).To(BeTrue(), "label not found!")
Expect(labelWildcardFound).To(BeTrue(), "label for wildcard nodename not found!")
Expect(labelNegativeFound).To(BeFalse(), "label for not existing nodename found!")
By("Deleting nfd-worker daemonset")
err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(context.TODO(), workerDS.ObjectMeta.Name, metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
cleanupNode(f.ClientSet)
})
})
})
})