From 3fff409f6d998f7642700bb36be013eec8583b65 Mon Sep 17 00:00:00 2001 From: AhmedGrati Date: Sun, 5 Mar 2023 22:56:46 +0100 Subject: [PATCH] Add master config file Similar to the nfd-worker, in this PR we want to support the dynamic run-time configurability through a config file for the nfd-master. We'll use a json or yaml configuration file along with the fsnotify in order to watch for changes in the config file. As a result, we're allowing dynamic control of logging params, allowed namespaces, extended resources, label whitelisting, and denied namespaces. Signed-off-by: AhmedGrati --- Makefile | 6 + cmd/nfd-master/main.go | 62 ++++-- deployment/base/master/master-deployment.yaml | 3 - .../components/common/kustomization.yaml | 4 + .../components/common/master-mounts.yaml | 13 ++ .../master-config/kustomization.yaml | 10 + .../master-config/nfd-master.conf.example | 6 + .../templates/master.yaml | 10 + .../templates/nfd-master-conf.yaml | 10 + .../helm/node-feature-discovery/values.yaml | 8 + .../default-combined/kustomization.yaml | 1 + .../overlays/default-job/kustomization.yaml | 1 + .../overlays/default/kustomization.yaml | 1 + .../kustomization.yaml | 1 + docs/deployment/helm.md | 1 + .../reference/master-commandline-reference.md | 28 +++ .../master-configuration-reference.md | 111 ++++++++++ .../topology-gc-commandline-reference.md | 2 +- .../topology-updater-commandline-reference.md | 2 +- ...opology-updater-configuration-reference.md | 2 +- .../worker-configuration-reference.md | 2 +- docs/usage/nfd-master.md | 33 +++ pkg/nfd-master/nfd-master-internal_test.go | 189 +++++++++++++++++- pkg/nfd-master/nfd-master.go | 186 +++++++++++++---- pkg/nfd-master/nfd-master_test.go | 6 + pkg/nfd-worker/nfd-worker_test.go | 7 +- test/e2e/node_feature_discovery_test.go | 106 ++++++++-- test/e2e/utils/configmap.go | 18 ++ 28 files changed, 739 insertions(+), 90 deletions(-) create mode 100644 deployment/components/common/master-mounts.yaml create mode 100644 deployment/components/master-config/kustomization.yaml create mode 100644 deployment/components/master-config/nfd-master.conf.example create mode 100644 deployment/helm/node-feature-discovery/templates/nfd-master-conf.yaml create mode 100644 docs/reference/master-configuration-reference.md diff --git a/Makefile b/Makefile index a1b60a3d9..b3f41b3c1 100644 --- a/Makefile +++ b/Makefile @@ -120,8 +120,13 @@ templates: @# Need to prepend each line in the sample config with spaces in order to @# fit correctly in the configmap spec. @sed s'/^/ /' deployment/components/worker-config/nfd-worker.conf.example > nfd-worker.conf.tmp + @sed s'/^/ /' deployment/components/master-config/nfd-master.conf.example > nfd-master.conf.tmp @sed s'/^/ /' deployment/components/topology-updater-config/nfd-topology-updater.conf.example > nfd-topology-updater.conf.tmp @# The sed magic below replaces the block of text between the lines with start and end markers + @start=NFD-MASTER-CONF-START-DO-NOT-REMOVE; \ + end=NFD-MASTER-CONF-END-DO-NOT-REMOVE; \ + sed -e "/$$start/,/$$end/{ /$$start/{ p; r nfd-master.conf.tmp" \ + -e "}; /$$end/p; d }" -i deployment/helm/node-feature-discovery/values.yaml @start=NFD-WORKER-CONF-START-DO-NOT-REMOVE; \ end=NFD-WORKER-CONF-END-DO-NOT-REMOVE; \ sed -e "/$$start/,/$$end/{ /$$start/{ p; r nfd-worker.conf.tmp" \ @@ -130,6 +135,7 @@ templates: end=NFD-TOPOLOGY-UPDATER-CONF-END-DO-NOT-REMOVE; \ sed -e "/$$start/,/$$end/{ /$$start/{ p; r nfd-topology-updater.conf.tmp" \ -e "}; /$$end/p; d }" -i deployment/helm/node-feature-discovery/values.yaml + @rm nfd-master.conf.tmp @rm nfd-worker.conf.tmp @rm nfd-topology-updater.conf.tmp diff --git a/cmd/nfd-master/main.go b/cmd/nfd-master/main.go index 7f1d582c8..58d0ca432 100644 --- a/cmd/nfd-master/main.go +++ b/cmd/nfd-master/main.go @@ -20,7 +20,6 @@ import ( "flag" "fmt" "os" - "regexp" "k8s.io/klog/v2" @@ -39,7 +38,7 @@ func main() { printVersion := flags.Bool("version", false, "Print version and exit.") - args := initFlags(flags) + args, overrides := initFlags(flags) // Inject klog flags klog.InitFlags(flags) @@ -55,6 +54,18 @@ func main() { switch f.Name { case "featurerules-controller": klog.Warningf("-featurerules-controller is deprecated, use '-crd-controller' flag instead") + case "extra-label-ns": + args.Overrides.ExtraLabelNs = overrides.ExtraLabelNs + case "deny-label-ns": + args.Overrides.DenyLabelNs = overrides.DenyLabelNs + case "label-whitelist": + args.Overrides.LabelWhiteList = overrides.LabelWhiteList + case "resource-labels": + args.Overrides.ResourceLabels = overrides.ResourceLabels + case "enable-taints": + args.Overrides.EnableTaints = overrides.EnableTaints + case "no-publish": + args.Overrides.NoPublish = overrides.NoPublish } }) @@ -82,35 +93,23 @@ func main() { } } -func initFlags(flagset *flag.FlagSet) *master.Args { - args := &master.Args{ - LabelWhiteList: utils.RegexpVal{Regexp: *regexp.MustCompile("")}, - DenyLabelNs: map[string]struct{}{"*.kubernetes.io": {}}, - } +func initFlags(flagset *flag.FlagSet) (*master.Args, *master.ConfigOverrideArgs) { + args := &master.Args{} flagset.StringVar(&args.CaFile, "ca-file", "", "Root certificate for verifying connections") flagset.StringVar(&args.CertFile, "cert-file", "", "Certificate used for authenticating connections") - flagset.Var(&args.DenyLabelNs, "deny-label-ns", - "Comma separated list of denied label namespaces") - flagset.Var(&args.ExtraLabelNs, "extra-label-ns", - "Comma separated list of allowed extra label namespaces") flagset.StringVar(&args.Instance, "instance", "", "Instance name. Used to separate annotation namespaces for multiple parallel deployments.") flagset.StringVar(&args.KeyFile, "key-file", "", "Private key matching -cert-file") + flagset.StringVar(&args.ConfigFile, "config", "/etc/kubernetes/node-feature-discovery/nfd-master.conf", + "Config file to use.") flagset.StringVar(&args.Kubeconfig, "kubeconfig", "", "Kubeconfig to use") - flagset.Var(&args.LabelWhiteList, "label-whitelist", - "Regular expression to filter label names to publish to the Kubernetes API server. "+ - "NB: the label namespace is omitted i.e. the filter is only applied to the name part after '/'.") flagset.BoolVar(&args.EnableNodeFeatureApi, "enable-nodefeature-api", false, "Enable the NodeFeature CRD API for receiving node features. This will automatically disable the gRPC communication.") - flagset.BoolVar(&args.NoPublish, "no-publish", false, - "Do not publish feature labels") - flagset.BoolVar(&args.EnableTaints, "enable-taints", false, - "Enable node tainting feature") flagset.BoolVar(&args.CrdController, "featurerules-controller", true, "Enable NFD CRD API controller. DEPRECATED: use -crd-controller instead") flagset.BoolVar(&args.CrdController, "crd-controller", true, @@ -119,11 +118,32 @@ func initFlags(flagset *flag.FlagSet) *master.Args { "Port on which to listen for connections.") flagset.BoolVar(&args.Prune, "prune", false, "Prune all NFD related attributes from all nodes of the cluaster and exit.") - flagset.Var(&args.ResourceLabels, "resource-labels", - "Comma separated list of labels to be exposed as extended resources.") flagset.BoolVar(&args.VerifyNodeName, "verify-node-name", false, "Verify worker node name against the worker's TLS certificate. "+ "Only takes effect when TLS authentication has been enabled.") + flagset.StringVar(&args.Options, "options", "", + "Specify config options from command line. Config options are specified "+ + "in the same format as in the config file (i.e. json or yaml). These options") - return args + overrides := &master.ConfigOverrideArgs{ + LabelWhiteList: &utils.RegexpVal{}, + DenyLabelNs: &utils.StringSetVal{}, + ExtraLabelNs: &utils.StringSetVal{}, + ResourceLabels: &utils.StringSetVal{}, + } + flagset.Var(overrides.ExtraLabelNs, "extra-label-ns", + "Comma separated list of allowed extra label namespaces") + flagset.Var(overrides.LabelWhiteList, "label-whitelist", + "Regular expression to filter label names to publish to the Kubernetes API server. "+ + "NB: the label namespace is omitted i.e. the filter is only applied to the name part after '/'.") + overrides.EnableTaints = flagset.Bool("enable-taints", false, + "Enable node tainting feature") + overrides.NoPublish = flagset.Bool("no-publish", false, + "Do not publish feature labels") + flagset.Var(overrides.DenyLabelNs, "deny-label-ns", + "Comma separated list of denied label namespaces") + flagset.Var(overrides.ResourceLabels, "resource-labels", + "Comma separated list of labels to be exposed as extended resources.") + + return args, overrides } diff --git a/deployment/base/master/master-deployment.yaml b/deployment/base/master/master-deployment.yaml index bb0e9979f..e52c91a56 100644 --- a/deployment/base/master/master-deployment.yaml +++ b/deployment/base/master/master-deployment.yaml @@ -34,6 +34,3 @@ spec: failureThreshold: 10 command: - "nfd-master" - args: [] - volumeMounts: [] - volumes: [] diff --git a/deployment/components/common/kustomization.yaml b/deployment/components/common/kustomization.yaml index 459b0cfa5..2776c6928 100644 --- a/deployment/components/common/kustomization.yaml +++ b/deployment/components/common/kustomization.yaml @@ -35,3 +35,7 @@ patches: target: labelSelector: app=nfd name: nfd +- path: master-mounts.yaml + target: + labelSelector: app=nfd + name: nfd-master diff --git a/deployment/components/common/master-mounts.yaml b/deployment/components/common/master-mounts.yaml new file mode 100644 index 000000000..750f4241d --- /dev/null +++ b/deployment/components/common/master-mounts.yaml @@ -0,0 +1,13 @@ +- op: add + path: /spec/template/spec/volumes + value: + - name: nfd-master-conf + configMap: + name: nfd-master-conf + +- op: add + path: /spec/template/spec/containers/0/volumeMounts + value: + - name: nfd-master-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true diff --git a/deployment/components/master-config/kustomization.yaml b/deployment/components/master-config/kustomization.yaml new file mode 100644 index 000000000..f042ce3d8 --- /dev/null +++ b/deployment/components/master-config/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- files: + - nfd-master.conf=nfd-master.conf.example + name: nfd-master-conf diff --git a/deployment/components/master-config/nfd-master.conf.example b/deployment/components/master-config/nfd-master.conf.example new file mode 100644 index 000000000..4f5ffffa3 --- /dev/null +++ b/deployment/components/master-config/nfd-master.conf.example @@ -0,0 +1,6 @@ +# noPublish: false +# extraLabelNs: ["added.ns.io","added.kubernets.io"] +# denyLabelNs: ["denied.ns.io","denied.kubernetes.io"] +# resourceLabels: ["vendor-1.com/feature-1","vendor-2.io/feature-2"] +# enableTaints: false +# labelWhiteList: "foo" diff --git a/deployment/helm/node-feature-discovery/templates/master.yaml b/deployment/helm/node-feature-discovery/templates/master.yaml index c259df529..0a6993bdb 100644 --- a/deployment/helm/node-feature-discovery/templates/master.yaml +++ b/deployment/helm/node-feature-discovery/templates/master.yaml @@ -109,10 +109,20 @@ spec: - name: nfd-master-cert mountPath: "/etc/kubernetes/node-feature-discovery/certs" readOnly: true + - name: nfd-master-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true volumes: - name: nfd-master-cert secret: secretName: nfd-master-cert + - name: nfd-master-conf + configMap: + name: {{ include "node-feature-discovery.fullname" . }}-master-conf + items: + - key: nfd-master.conf + path: nfd-master.conf + ## /TLS ## {{- end }} {{- with .Values.master.nodeSelector }} diff --git a/deployment/helm/node-feature-discovery/templates/nfd-master-conf.yaml b/deployment/helm/node-feature-discovery/templates/nfd-master-conf.yaml new file mode 100644 index 000000000..c806a8e5d --- /dev/null +++ b/deployment/helm/node-feature-discovery/templates/nfd-master-conf.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-master-conf + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +data: + nfd-master.conf: |- + {{- .Values.master.config | toYaml | nindent 4 }} diff --git a/deployment/helm/node-feature-discovery/values.yaml b/deployment/helm/node-feature-discovery/values.yaml index 061476f7b..4ef23e83d 100644 --- a/deployment/helm/node-feature-discovery/values.yaml +++ b/deployment/helm/node-feature-discovery/values.yaml @@ -13,6 +13,14 @@ namespaceOverride: "" enableNodeFeatureApi: false master: + config: ### + # noPublish: false + # extraLabelNs: ["added.ns.io","added.kubernets.io"] + # denyLabelNs: ["denied.ns.io","denied.kubernetes.io"] + # resourceLabels: ["vendor-1.com/feature-1","vendor-2.io/feature-2"] + # enableTaints: false + # labelWhiteList: "foo" + ### # The TCP port that nfd-master listens for incoming requests. Default: 8080 port: 8080 instance: diff --git a/deployment/overlays/default-combined/kustomization.yaml b/deployment/overlays/default-combined/kustomization.yaml index a7778fc83..d980ea0b0 100644 --- a/deployment/overlays/default-combined/kustomization.yaml +++ b/deployment/overlays/default-combined/kustomization.yaml @@ -14,3 +14,4 @@ resources: components: - ../../components/worker-config - ../../components/common +- ../../components/master-config diff --git a/deployment/overlays/default-job/kustomization.yaml b/deployment/overlays/default-job/kustomization.yaml index 04bfd7688..8d2850000 100644 --- a/deployment/overlays/default-job/kustomization.yaml +++ b/deployment/overlays/default-job/kustomization.yaml @@ -15,3 +15,4 @@ resources: components: - ../../components/worker-config - ../../components/common +- ../../components/master-config diff --git a/deployment/overlays/default/kustomization.yaml b/deployment/overlays/default/kustomization.yaml index 317095e79..8774b59f8 100644 --- a/deployment/overlays/default/kustomization.yaml +++ b/deployment/overlays/default/kustomization.yaml @@ -15,3 +15,4 @@ resources: components: - ../../components/worker-config - ../../components/common +- ../../components/master-config diff --git a/deployment/overlays/master-worker-topologyupdater/kustomization.yaml b/deployment/overlays/master-worker-topologyupdater/kustomization.yaml index ebc3c1fc0..ebf0ac213 100644 --- a/deployment/overlays/master-worker-topologyupdater/kustomization.yaml +++ b/deployment/overlays/master-worker-topologyupdater/kustomization.yaml @@ -22,3 +22,4 @@ components: - ../../components/common - ../../components/topology-updater - ../../components/topology-updater-config +- ../../components/master-config diff --git a/docs/deployment/helm.md b/docs/deployment/helm.md index d28a94304..a89d69ed9 100644 --- a/docs/deployment/helm.md +++ b/docs/deployment/helm.md @@ -129,6 +129,7 @@ We have introduced the following Chart parameters. | `master.annotations` | dict | {} | NFD master pod [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) | | `master.affinity` | dict | | NFD master pod required [node affinity](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/) | | `master.deploymentAnnotations` | dict | {} | NFD master deployment [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) | +| `master.config` | dict | | NFD master [configuration](../reference/master-configuration-reference) | ### Worker pod parameters diff --git a/docs/reference/master-commandline-reference.md b/docs/reference/master-commandline-reference.md index a62f348d1..bf8cc7787 100644 --- a/docs/reference/master-commandline-reference.md +++ b/docs/reference/master-commandline-reference.md @@ -268,6 +268,34 @@ Example: nfd-master -resource-labels=vendor-1.com/feature-1,vendor-2.io/feature-2 ``` +### -config + +The `-config` flag specifies the path of the nfd-master configuration file to +use. + +Default: /etc/kubernetes/node-feature-discovery/nfd-master.conf + +Example: + +```bash +nfd-master -config=/opt/nfd/master.conf +``` + +### -options + +The `-options` flag may be used to specify and override configuration file +options directly from the command line. The required format is the same as in +the config file i.e. JSON or YAML. Configuration options specified via this +flag will override those from the configuration file: + +Default: *empty* + +Example: + +```bash +nfd-master -options='{"noPublish": true}' +``` + ### Logging The following logging-related flags are inherited from the diff --git a/docs/reference/master-configuration-reference.md b/docs/reference/master-configuration-reference.md new file mode 100644 index 000000000..7957d4ab5 --- /dev/null +++ b/docs/reference/master-configuration-reference.md @@ -0,0 +1,111 @@ +--- +title: "Master config reference" +layout: default +sort: 3 +--- + +# Configuration file reference of nfd-master +{: .no_toc} + +## Table of contents +{: .no_toc .text-delta} + +1. TOC +{:toc} + +--- + +See the +[sample configuration file](https://github.com/kubernetes-sigs/node-feature-discovery/blob/{{site.release}}/deployment/components/master-config/nfd-master.conf.example) +for a full example configuration. + +## noPublish + +`noPublish` option disables updates to the Node objects in the Kubernetes +API server, making a "dry-run" flag for nfd-master. No Labels, Annotations, Taints +or ExtendedResources of nodes are updated. + +Default: `false` + +Example: + +```yaml +noPublish: true +``` + +## extraLabelNs +`extraLabelNs` specifies a list of allowed feature +label namespaces. This option can be used to allow +other vendor or application specific namespaces for custom labels from the +local and custom feature sources, even though these labels were denied using +the `denyLabelNs` parameter. + +The same namespace control and this option applies to Extended Resources (created +with `resourceLabels`), too. + +Default: *empty* + +Example: + +```yaml +extraLabelNs: ["added.ns.io","added.kubernets.io"] +``` + +## denyLabelNs +`denyLabelNs` specifies a list of excluded +label namespaces. By default, nfd-master allows creating labels in all +namespaces, excluding `kubernetes.io` namespace and its sub-namespaces +(i.e. `*.kubernetes.io`). However, you should note that +`kubernetes.io` and its sub-namespaces are always denied. +This option can be used to exclude some vendors or application specific +namespaces. +Note that the namespaces `feature.node.kubernetes.io` and `profile.node.kubernetes.io` +and their sub-namespaces are always allowed and cannot be denied. + +Default: *empty* + +Example: + +```yaml +denyLabelNs: ["denied.ns.io","denied.kubernetes.io"] +``` + +## resourceLabels +The `resourceLabels` option specifies a list of features to be +advertised as extended resources instead of labels. Features that have integer +values can be published as Extended Resources by listing them in this option. + +Default: *empty* + +Example: + +```yaml +resourceLabels: ["vendor-1.com/feature-1","vendor-2.io/feature-2"] +``` + +## enableTaints +`enableTaints` enables/disables node tainting feature of NFD. + +Default: *false* + +Example: + +```yaml +enableTaints: true +``` + +## labelWhiteList +`labelWhiteList` specifies a regular expression for filtering feature +labels based on their name. Each label must match against the given reqular +expression in order to be published. + +Note: The regular expression is only matches against the "basename" part of the +label, i.e. to the part of the name after '/'. The label namespace is omitted. + +Default: *empty* + +Example: + +```yaml +labelWhiteList: "foo" +``` diff --git a/docs/reference/topology-gc-commandline-reference.md b/docs/reference/topology-gc-commandline-reference.md index bfabb1f19..583dbbcc9 100644 --- a/docs/reference/topology-gc-commandline-reference.md +++ b/docs/reference/topology-gc-commandline-reference.md @@ -1,7 +1,7 @@ --- title: "Topology Garbage Collector Cmdline Reference" layout: default -sort: 6 +sort: 7 --- # NFD-Topology-Garbage-Collector Commandline Flags diff --git a/docs/reference/topology-updater-commandline-reference.md b/docs/reference/topology-updater-commandline-reference.md index aa67ac665..459643c75 100644 --- a/docs/reference/topology-updater-commandline-reference.md +++ b/docs/reference/topology-updater-commandline-reference.md @@ -1,7 +1,7 @@ --- title: "Topology Updater Cmdline Reference" layout: default -sort: 4 +sort: 5 --- # NFD-Topology-Updater Commandline Flags diff --git a/docs/reference/topology-updater-configuration-reference.md b/docs/reference/topology-updater-configuration-reference.md index cda081452..6660a1587 100644 --- a/docs/reference/topology-updater-configuration-reference.md +++ b/docs/reference/topology-updater-configuration-reference.md @@ -1,7 +1,7 @@ --- title: "Topology-Updater config reference" layout: default -sort: 5 +sort: 6 --- # Configuration file reference of nfd-topology-updater diff --git a/docs/reference/worker-configuration-reference.md b/docs/reference/worker-configuration-reference.md index 4d445f3a0..540172caf 100644 --- a/docs/reference/worker-configuration-reference.md +++ b/docs/reference/worker-configuration-reference.md @@ -1,7 +1,7 @@ --- title: "Worker config reference" layout: default -sort: 3 +sort: 4 --- # Configuration file reference of nfd-worker diff --git a/docs/usage/nfd-master.md b/docs/usage/nfd-master.md index 4fac00d74..60d009e9a 100644 --- a/docs/usage/nfd-master.md +++ b/docs/usage/nfd-master.md @@ -52,6 +52,39 @@ enabled. > present when gRPC interface is disabled > and [NodeFeature](custom-resources.md#nodefeature-custom-resource) API is used. +## Master configuration + +NFD-Master supports dynamic configuration through a configuration file. The +default location is `/etc/kubernetes/node-feature-discovery/nfd-master.conf`, +but, this can be changed by specifying the`-config` command line flag. +Configuration file is re-read whenever it is modified which makes run-time +re-configuration of nfd-master straightforward. + +Master configuration file is read inside the container, and thus, Volumes and +VolumeMounts are needed to make your configuration available for NFD. The +preferred method is to use a ConfigMap which provides easy deployment and +re-configurability. + +The provided nfd-master deployment templates create an empty configmap and +mount it inside the nfd-master containers. In kustomize deployments, +configuration can be edited with: + +```bash +kubectl -n ${NFD_NS} edit configmap nfd-master-conf +``` + +In Helm deployments, +[Master pod parameter](../deployment/helm.md#master-pod-parameters) +`master.config` can be used to edit the respective configuration. + +See +[nfd-master configuration file reference](../reference/master-configuration-reference.md) +for more details. +The (empty-by-default) +[example config](https://github.com/kubernetes-sigs/node-feature-discovery/blob/{{site.release}}/deployment/components/master-config/nfd-master.conf.example) +contains all available configuration options and can be used as a reference +for creating a configuration. + ## Deployment notes NFD-Master runs as a deployment, by default diff --git a/pkg/nfd-master/nfd-master-internal_test.go b/pkg/nfd-master/nfd-master-internal_test.go index 2281e8f20..1832efc80 100644 --- a/pkg/nfd-master/nfd-master-internal_test.go +++ b/pkg/nfd-master/nfd-master-internal_test.go @@ -18,10 +18,13 @@ package nfdmaster import ( "fmt" + "os" + "path/filepath" "regexp" "sort" "strings" "testing" + "time" "github.com/smartystreets/assertions" . "github.com/smartystreets/goconvey/convey" @@ -56,7 +59,7 @@ func newMockNode() *corev1.Node { func newMockMaster(apihelper apihelper.APIHelpers) *nfdMaster { return &nfdMaster{ nodeName: mockNodeName, - args: Args{LabelWhiteList: utils.RegexpVal{Regexp: *regexp.MustCompile("")}}, + config: &NFDConfig{LabelWhiteList: utils.RegexpVal{Regexp: *regexp.MustCompile("")}}, apihelper: apihelper, } } @@ -341,7 +344,7 @@ func TestSetLabels(t *testing.T) { apihelper.NewJsonPatch("add", "/metadata/labels", nfdv1alpha1.FeatureLabelNs+"/feature-2", mockLabels["feature-2"]), } - mockMaster.args.LabelWhiteList.Regexp = *regexp.MustCompile("^f.*2$") + mockMaster.config.LabelWhiteList.Regexp = *regexp.MustCompile("^f.*2$") mockHelper.On("GetClient").Return(mockClient, nil) mockHelper.On("GetNode", mockClient, workerName).Return(mockNode, nil) mockHelper.On("PatchNode", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(expectedPatches))).Return(nil) @@ -378,7 +381,7 @@ func TestSetLabels(t *testing.T) { mockMaster.deniedNs.normal = map[string]struct{}{"random.denied.ns": {}} mockMaster.deniedNs.wildcard = map[string]struct{}{"kubernetes.io": {}} - mockMaster.args.ExtraLabelNs = map[string]struct{}{"valid.ns": {}} + mockMaster.config.ExtraLabelNs = map[string]struct{}{"valid.ns": {}} mockMaster.args.Instance = instance mockHelper.On("GetClient").Return(mockClient, nil) mockHelper.On("GetNode", mockClient, workerName).Return(mockNode, nil) @@ -404,7 +407,7 @@ func TestSetLabels(t *testing.T) { apihelper.NewJsonPatch("add", "/status/capacity", nfdv1alpha1.FeatureLabelNs+"/feature-3", mockLabels["feature-3"]), } - mockMaster.args.ResourceLabels = map[string]struct{}{"feature-3": {}, "feature-1": {}} + mockMaster.config.ResourceLabels = map[string]struct{}{"feature-3": {}, "feature-1": {}} mockHelper.On("GetClient").Return(mockClient, nil) mockHelper.On("GetNode", mockClient, workerName).Return(mockNode, nil) mockHelper.On("PatchNode", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(expectedPatches))).Return(nil) @@ -424,7 +427,7 @@ func TestSetLabels(t *testing.T) { }) }) - mockMaster.args.NoPublish = true + mockMaster.config.NoPublish = true Convey("With '-no-publish'", func() { _, err := mockMaster.SetLabels(mockCtx, mockReq) Convey("Operation should succeed", func() { @@ -513,6 +516,182 @@ func TestRemoveLabelsWithPrefix(t *testing.T) { }) } +func TestConfigParse(t *testing.T) { + Convey("When parsing configuration", t, func() { + m, err := NewNfdMaster(&Args{}) + So(err, ShouldBeNil) + master := m.(*nfdMaster) + overrides := `{"noPublish": true, "enableTaints": true, "extraLabelNs": ["added.ns.io","added.kubernetes.io"], "denyLabelNs": ["denied.ns.io","denied.kubernetes.io"], "resourceLabels": ["vendor-1.com/feature-1","vendor-2.io/feature-2"], "labelWhiteList": "foo"}` + + Convey("and no core cmdline flags have been specified", func() { + So(master.configure("non-existing-file", overrides), ShouldBeNil) + Convey("overrides should be in effect", func() { + So(master.config.NoPublish, ShouldResemble, true) + So(master.config.EnableTaints, ShouldResemble, true) + So(master.config.ExtraLabelNs, ShouldResemble, utils.StringSetVal{"added.ns.io": struct{}{}, "added.kubernetes.io": struct{}{}}) + So(master.config.DenyLabelNs, ShouldResemble, utils.StringSetVal{"denied.ns.io": struct{}{}, "denied.kubernetes.io": struct{}{}}) + So(master.config.ResourceLabels, ShouldResemble, utils.StringSetVal{"vendor-1.com/feature-1": struct{}{}, "vendor-2.io/feature-2": struct{}{}}) + So(master.config.LabelWhiteList.String(), ShouldEqual, "foo") + }) + }) + Convey("and a non-accessible file, but cmdline flags and some overrides are specified", func() { + master.args = Args{Overrides: ConfigOverrideArgs{ + ExtraLabelNs: &utils.StringSetVal{"override.added.ns.io": struct{}{}}, + DenyLabelNs: &utils.StringSetVal{"override.denied.ns.io": struct{}{}}}} + So(master.configure("non-existing-file", overrides), ShouldBeNil) + + Convey("cmdline flags should be in effect instead overrides", func() { + So(master.config.ExtraLabelNs, ShouldResemble, utils.StringSetVal{"override.added.ns.io": struct{}{}}) + So(master.config.DenyLabelNs, ShouldResemble, utils.StringSetVal{"override.denied.ns.io": struct{}{}}) + }) + Convey("overrides should take effect", func() { + So(master.config.NoPublish, ShouldBeTrue) + So(master.config.EnableTaints, ShouldBeTrue) + }) + }) + // Create a temporary config file + f, err := os.CreateTemp("", "nfd-test-") + defer os.Remove(f.Name()) + So(err, ShouldBeNil) + _, err = f.WriteString(` +noPublish: true +denyLabelNs: ["denied.ns.io","denied.kubernetes.io"] +resourceLabels: ["vendor-1.com/feature-1","vendor-2.io/feature-2"] +enableTaints: false +labelWhiteList: "foo" +`) + f.Close() + So(err, ShouldBeNil) + + Convey("and a proper config file is specified", func() { + master.args = Args{Overrides: ConfigOverrideArgs{ExtraLabelNs: &utils.StringSetVal{"override.added.ns.io": struct{}{}}}} + So(master.configure(f.Name(), ""), ShouldBeNil) + Convey("specified configuration should take effect", func() { + // Verify core config + So(master.config.NoPublish, ShouldBeTrue) + So(master.config.EnableTaints, ShouldBeFalse) + So(master.config.ExtraLabelNs, ShouldResemble, utils.StringSetVal{"override.added.ns.io": struct{}{}}) + So(master.config.ResourceLabels, ShouldResemble, utils.StringSetVal{"vendor-1.com/feature-1": struct{}{}, "vendor-2.io/feature-2": struct{}{}}) // from cmdline + So(master.config.DenyLabelNs, ShouldResemble, utils.StringSetVal{"denied.ns.io": struct{}{}, "denied.kubernetes.io": struct{}{}}) + So(master.config.LabelWhiteList.String(), ShouldEqual, "foo") + }) + }) + + Convey("and a proper config file and overrides are given", func() { + master.args = Args{Overrides: ConfigOverrideArgs{DenyLabelNs: &utils.StringSetVal{"denied.ns.io": struct{}{}}}} + overrides := `{"extraLabelNs": ["added.ns.io"], "noPublish": true}` + So(master.configure(f.Name(), overrides), ShouldBeNil) + + Convey("overrides should take precedence over the config file", func() { + // Verify core config + So(master.config.ExtraLabelNs, ShouldResemble, utils.StringSetVal{"added.ns.io": struct{}{}}) // from overrides + So(master.config.DenyLabelNs, ShouldResemble, utils.StringSetVal{"denied.ns.io": struct{}{}}) // from cmdline + }) + }) + }) +} + +func TestDynamicConfig(t *testing.T) { + Convey("When running nfd-master", t, func() { + tmpDir, err := os.MkdirTemp("", "*.nfd-test") + So(err, ShouldBeNil) + defer os.RemoveAll(tmpDir) + + // Create (temporary) dir for config + configDir := filepath.Join(tmpDir, "subdir-1", "subdir-2", "master.conf") + err = os.MkdirAll(configDir, 0755) + So(err, ShouldBeNil) + + // Create config file + configFile := filepath.Join(configDir, "master.conf") + + writeConfig := func(data string) { + f, err := os.Create(configFile) + So(err, ShouldBeNil) + _, err = f.WriteString(data) + So(err, ShouldBeNil) + err = f.Close() + So(err, ShouldBeNil) + } + writeConfig(` +extraLabelNs: ["added.ns.io"] +`) + + noPublish := true + m, err := NewNfdMaster(&Args{ + ConfigFile: configFile, + Overrides: ConfigOverrideArgs{ + NoPublish: &noPublish, + }, + }) + So(err, ShouldBeNil) + master := m.(*nfdMaster) + + Convey("config file updates should take effect", func() { + go func() { _ = m.Run() }() + defer m.Stop() + // Check initial config + time.Sleep(10 * time.Second) + So(func() interface{} { return master.config.ExtraLabelNs }, + withTimeout, 2*time.Second, ShouldResemble, utils.StringSetVal{"added.ns.io": struct{}{}}) + + // Update config and verify the effect + writeConfig(` +extraLabelNs: ["override.ns.io"] +`) + So(func() interface{} { return master.config.ExtraLabelNs }, + withTimeout, 2*time.Second, ShouldResemble, utils.StringSetVal{"override.ns.io": struct{}{}}) + + // Removing config file should get back our defaults + err = os.RemoveAll(tmpDir) + So(err, ShouldBeNil) + So(func() interface{} { return master.config.ExtraLabelNs }, + withTimeout, 2*time.Second, ShouldResemble, utils.StringSetVal{}) + + // Re-creating config dir and file should change the config + err = os.MkdirAll(configDir, 0755) + So(err, ShouldBeNil) + writeConfig(` +extraLabelNs: ["another.override.ns"] +`) + So(func() interface{} { return master.config.ExtraLabelNs }, + withTimeout, 2*time.Second, ShouldResemble, utils.StringSetVal{"another.override.ns": struct{}{}}) + }) + }) +} + +// withTimeout is a custom assertion for polling a value asynchronously +// actual is a function for getting the actual value +// expected[0] is a time.Duration value specifying the timeout +// expected[1] is the "real" assertion function to be called +// expected[2:] are the arguments for the "real" assertion function +func withTimeout(actual interface{}, expected ...interface{}) string { + getter, ok := actual.(func() interface{}) + if !ok { + return "not getterFunc" + } + t, ok := expected[0].(time.Duration) + if !ok { + return "not time.Duration" + } + f, ok := expected[1].(func(interface{}, ...interface{}) string) + if !ok { + return "not an assert func" + } + timeout := time.After(t) + for { + result := f(getter(), expected[2:]...) + if result == "" { + return "" + } + select { + case <-timeout: + return result + case <-time.After(10 * time.Millisecond): + } + } +} + func jsonPatchMatcher(expected []apihelper.JsonPatch) func([]apihelper.JsonPatch) bool { return func(actual []apihelper.JsonPatch) bool { // We don't care about modifying the original slices diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index 288773e68..cb8b9a03f 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -21,7 +21,9 @@ import ( "crypto/x509" "fmt" "net" + "os" "path" + "path/filepath" "regexp" "sort" "strconv" @@ -42,6 +44,7 @@ import ( "k8s.io/klog/v2" controller "k8s.io/kubernetes/pkg/controller" taintutils "k8s.io/kubernetes/pkg/util/taints" + "sigs.k8s.io/yaml" "sigs.k8s.io/node-feature-discovery/pkg/apihelper" nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" @@ -59,24 +62,42 @@ type ExtendedResources map[string]string // Annotations are used for NFD-related node metadata type Annotations map[string]string +// NFDConfig contains the configuration settings of NfdMaster. +type NFDConfig struct { + DenyLabelNs utils.StringSetVal + ExtraLabelNs utils.StringSetVal + LabelWhiteList utils.RegexpVal + NoPublish bool + ResourceLabels utils.StringSetVal + EnableTaints bool +} + +// ConfigOverrideArgs are args that override config file options +type ConfigOverrideArgs struct { + DenyLabelNs *utils.StringSetVal + ExtraLabelNs *utils.StringSetVal + LabelWhiteList *utils.RegexpVal + ResourceLabels *utils.StringSetVal + EnableTaints *bool + NoPublish *bool +} + // Args holds command line arguments type Args struct { CaFile string CertFile string - DenyLabelNs utils.StringSetVal - ExtraLabelNs utils.StringSetVal + ConfigFile string Instance string KeyFile string Kubeconfig string - LabelWhiteList utils.RegexpVal CrdController bool EnableNodeFeatureApi bool - NoPublish bool - EnableTaints bool Port int Prune bool VerifyNodeName bool - ResourceLabels utils.StringSetVal + Options string + + Overrides ConfigOverrideArgs } type deniedNs struct { @@ -93,15 +114,17 @@ type NfdMaster interface { type nfdMaster struct { *nfdController - args Args - namespace string - nodeName string - server *grpc.Server - stop chan struct{} - ready chan bool - apihelper apihelper.APIHelpers - kubeconfig *restclient.Config + args Args + namespace string + nodeName string + configFilePath string + server *grpc.Server + stop chan struct{} + ready chan bool + apihelper apihelper.APIHelpers + kubeconfig *restclient.Config deniedNs + config *NFDConfig } // NewNfdMaster creates a new NfdMaster server instance. @@ -133,29 +156,25 @@ func NewNfdMaster(args *Args) (NfdMaster, error) { return nfd, fmt.Errorf("-ca-file needs to be specified alongside -cert-file and -key-file") } } - if args.DenyLabelNs == nil { - args.DenyLabelNs = make(utils.StringSetVal) - } - // Pre-process DenyLabelNS into 2 lists: one for normal ns, and the other for wildcard ns - normalDeniedNs, wildcardDeniedNs := preProcessDeniedNamespaces(args.DenyLabelNs) - nfd.deniedNs.normal = normalDeniedNs - nfd.deniedNs.wildcard = wildcardDeniedNs - // We forcibly deny kubernetes.io - nfd.deniedNs.normal["kubernetes.io"] = struct{}{} - nfd.deniedNs.wildcard[".kubernetes.io"] = struct{}{} - // Initialize Kubernetes API helpers - if !args.NoPublish { - kubeconfig, err := nfd.getKubeconfig() - if err != nil { - return nfd, err - } - nfd.apihelper = apihelper.K8sHelpers{Kubeconfig: kubeconfig} + if args.ConfigFile != "" { + nfd.configFilePath = filepath.Clean(args.ConfigFile) } return nfd, nil } +func newDefaultConfig() *NFDConfig { + return &NFDConfig{ + LabelWhiteList: utils.RegexpVal{Regexp: *regexp.MustCompile("")}, + DenyLabelNs: utils.StringSetVal{}, + ExtraLabelNs: utils.StringSetVal{}, + NoPublish: false, + ResourceLabels: utils.StringSetVal{}, + EnableTaints: false, + } +} + // Run NfdMaster server. The method returns in case of fatal errors or if Stop() // is called. func (m *nfdMaster) Run() error { @@ -182,13 +201,21 @@ func (m *nfdMaster) Run() error { } } - if !m.args.NoPublish { + // Create watcher for config file and read initial configuration + configWatch, err := utils.CreateFsWatcher(time.Second, m.configFilePath) + if err != nil { + return err + } + if err := m.configure(m.configFilePath, m.args.Options); err != nil { + return err + } + + if !m.config.NoPublish { err := m.updateMasterNode() if err != nil { return fmt.Errorf("failed to update master node: %v", err) } } - // Run gRPC server grpcErr := make(chan error, 1) go m.runGrpcServer(grpcErr) @@ -208,6 +235,15 @@ func (m *nfdMaster) Run() error { case err := <-grpcErr: return fmt.Errorf("error in serving gRPC: %w", err) + case <-configWatch.Events: + klog.Infof("reloading configuration") + if err := m.configure(m.configFilePath, m.args.Options); err != nil { + return err + } + // Update all nodes when the configuration changes + if m.nfdController != nil { + m.nfdController.updateAllNodesChan <- struct{}{} + } case <-m.stop: klog.Infof("shutting down nfd-master") return nil @@ -423,7 +459,7 @@ func (m *nfdMaster) filterFeatureLabels(labels Labels) (Labels, ExtendedResource !strings.HasSuffix(ns, nfdv1alpha1.FeatureLabelSubNsSuffix) && !strings.HasSuffix(ns, nfdv1alpha1.ProfileLabelSubNsSuffix) { // If the namespace is denied, and not present in the extraLabelNs, label will be ignored if isNamespaceDenied(ns, m.deniedNs.wildcard, m.deniedNs.normal) { - if _, ok := m.args.ExtraLabelNs[ns]; !ok { + if _, ok := m.config.ExtraLabelNs[ns]; !ok { klog.Errorf("Namespace %q is not allowed. Ignoring label %q\n", ns, label) continue } @@ -431,8 +467,8 @@ func (m *nfdMaster) filterFeatureLabels(labels Labels) (Labels, ExtendedResource } // Skip if label doesn't match labelWhiteList - if !m.args.LabelWhiteList.Regexp.MatchString(name) { - klog.Errorf("%s (%s) does not match the whitelist (%s) and will not be published.", name, label, m.args.LabelWhiteList.Regexp.String()) + if !m.config.LabelWhiteList.Regexp.MatchString(name) { + klog.Errorf("%s (%s) does not match the whitelist (%s) and will not be published.", name, label, m.config.LabelWhiteList.Regexp.String()) continue } outLabels[label] = value @@ -440,7 +476,7 @@ func (m *nfdMaster) filterFeatureLabels(labels Labels) (Labels, ExtendedResource // Remove labels which are intended to be extended resources extendedResources := ExtendedResources{} - for extendedResourceName := range m.args.ResourceLabels { + for extendedResourceName := range m.config.ResourceLabels { // Add possibly missing default ns extendedResourceName = addNs(extendedResourceName, nfdv1alpha1.FeatureLabelNs) if value, ok := outLabels[extendedResourceName]; ok { @@ -498,8 +534,7 @@ func (m *nfdMaster) SetLabels(c context.Context, r *pb.SetLabelsRequest) (*pb.Se default: klog.Infof("received labeling request for node %q", r.NodeName) } - - if !m.args.NoPublish { + if !m.config.NoPublish { cli, err := m.apihelper.GetClient() if err != nil { return &pb.SetLabelsReply{}, err @@ -562,7 +597,7 @@ func (m *nfdMaster) nfdAPIUpdateOneNode(nodeName string) error { return objs[i].Namespace < objs[j].Namespace }) - if m.args.NoPublish { + if m.config.NoPublish { return nil } @@ -620,7 +655,7 @@ func (m *nfdMaster) refreshNodeFeatures(cli *kubernetes.Clientset, nodeName stri labels, extendedResources := m.filterFeatureLabels(labels) var taints []corev1.Taint - if m.args.EnableTaints { + if m.config.EnableTaints { taints = crTaints } @@ -921,6 +956,75 @@ func (m *nfdMaster) createExtendedResourcePatches(n *corev1.Node, extendedResour return patches } +// Parse configuration options +func (m *nfdMaster) configure(filepath string, overrides string) error { + // Create a new default config + c := newDefaultConfig() + + // Try to read and parse config file + if filepath != "" { + data, err := os.ReadFile(filepath) + if err != nil { + if os.IsNotExist(err) { + klog.Infof("config file %q not found, using defaults", filepath) + } else { + return fmt.Errorf("error reading config file: %s", err) + } + } else { + err = yaml.Unmarshal(data, c) + if err != nil { + return fmt.Errorf("failed to parse config file: %s", err) + } + + klog.Infof("configuration file %q parsed", filepath) + } + } + + // Parse config overrides + if err := yaml.Unmarshal([]byte(overrides), c); err != nil { + return fmt.Errorf("failed to parse -options: %s", err) + } + if m.args.Overrides.NoPublish != nil { + c.NoPublish = *m.args.Overrides.NoPublish + } + if m.args.Overrides.DenyLabelNs != nil { + c.DenyLabelNs = *m.args.Overrides.DenyLabelNs + } + if m.args.Overrides.ExtraLabelNs != nil { + c.ExtraLabelNs = *m.args.Overrides.ExtraLabelNs + } + if m.args.Overrides.ResourceLabels != nil { + c.ResourceLabels = *m.args.Overrides.ResourceLabels + } + if m.args.Overrides.EnableTaints != nil { + c.EnableTaints = *m.args.Overrides.EnableTaints + } + if m.args.Overrides.LabelWhiteList != nil { + c.LabelWhiteList = *m.args.Overrides.LabelWhiteList + } + + m.config = c + if !c.NoPublish { + kubeconfig, err := m.getKubeconfig() + if err != nil { + return err + } + m.apihelper = apihelper.K8sHelpers{Kubeconfig: kubeconfig} + } + // Pre-process DenyLabelNS into 2 lists: one for normal ns, and the other for wildcard ns + normalDeniedNs, wildcardDeniedNs := preProcessDeniedNamespaces(c.DenyLabelNs) + m.deniedNs.normal = normalDeniedNs + m.deniedNs.wildcard = wildcardDeniedNs + // We forcibly deny kubernetes.io + m.deniedNs.normal["kubernetes.io"] = struct{}{} + m.deniedNs.wildcard[".kubernetes.io"] = struct{}{} + + utils.KlogDump(1, "effective configuration:", " ", m.config) + klog.Infof("master (re-)configuration successfully completed") + + return nil +} + // addNs adds a namespace if one isn't already found from src string func addNs(src string, nsToAdd string) string { if strings.Contains(src, "/") { diff --git a/pkg/nfd-master/nfd-master_test.go b/pkg/nfd-master/nfd-master_test.go index a2cca826a..b15721601 100644 --- a/pkg/nfd-master/nfd-master_test.go +++ b/pkg/nfd-master/nfd-master_test.go @@ -35,5 +35,11 @@ func TestNewNfdMaster(t *testing.T) { So(err3, ShouldNotBeNil) }) }) + Convey("When -config is supplied", func() { + _, err := m.NewNfdMaster(&m.Args{CertFile: "crt", KeyFile: "key", CaFile: "ca", ConfigFile: "master-config.yaml"}) + Convey("An error should not be returned", func() { + So(err, ShouldBeNil) + }) + }) }) } diff --git a/pkg/nfd-worker/nfd-worker_test.go b/pkg/nfd-worker/nfd-worker_test.go index b80aa27b3..4a910338c 100644 --- a/pkg/nfd-worker/nfd-worker_test.go +++ b/pkg/nfd-worker/nfd-worker_test.go @@ -38,9 +38,12 @@ type testContext struct { func setupTest(args *master.Args) testContext { // Fixed port and no-publish, for convenience - args.NoPublish = true + publish := true + args.Overrides = master.ConfigOverrideArgs{ + NoPublish: &publish, + LabelWhiteList: &utils.RegexpVal{Regexp: *regexp.MustCompile("")}, + } args.Port = 8192 - args.LabelWhiteList.Regexp = *regexp.MustCompile("") m, err := master.NewNfdMaster(args) if err != nil { fmt.Printf("Test setup failed: %v\n", err) diff --git a/test/e2e/node_feature_discovery_test.go b/test/e2e/node_feature_discovery_test.go index b06302e0c..7ba2d6156 100644 --- a/test/e2e/node_feature_discovery_test.go +++ b/test/e2e/node_feature_discovery_test.go @@ -169,9 +169,10 @@ var _ = SIGDescribe("NFD master and worker", func() { Context("when deploying a single nfd-master pod", Ordered, func() { var ( - crds []*apiextensionsv1.CustomResourceDefinition - extClient *extclient.Clientset - nfdClient *nfdclient.Clientset + crds []*apiextensionsv1.CustomResourceDefinition + extClient *extclient.Clientset + nfdClient *nfdclient.Clientset + customMasterPodSpecOpts *[]testpod.SpecOption ) checkNodeFeatureObject := func(name string) { @@ -203,7 +204,7 @@ var _ = SIGDescribe("NFD master and worker", func() { } }) - BeforeEach(func() { + JustBeforeEach(func() { // Drop the pod security admission label as nfd-worker needs host mounts if _, ok := f.Namespace.Labels[admissionapi.EnforceLevelLabel]; ok { e2elog.Logf("Deleting %s label from the test namespace", admissionapi.EnforceLevelLabel) @@ -221,15 +222,21 @@ var _ = SIGDescribe("NFD master and worker", func() { // Launch nfd-master By("Creating nfd master pod and nfd-master service") - podSpecOpts := createPodSpecOpts( - testpod.SpecWithContainerImage(dockerImage()), - testpod.SpecWithTolerations(testTolerations), - testpod.SpecWithContainerExtraArgs("-enable-taints"), - testpod.SpecWithContainerExtraArgs( - "-deny-label-ns=*.denied.ns,random.unwanted.ns,*.vendor.io", - "-extra-label-ns=custom.vendor.io", - ), - ) + var podSpecOpts []testpod.SpecOption + if customMasterPodSpecOpts == nil { + podSpecOpts = createPodSpecOpts( + testpod.SpecWithContainerImage(dockerImage()), + testpod.SpecWithTolerations(testTolerations), + testpod.SpecWithContainerExtraArgs("-enable-taints"), + testpod.SpecWithContainerExtraArgs( + "-deny-label-ns=*.denied.ns,random.unwanted.ns,*.vendor.io", + "-extra-label-ns=custom.vendor.io", + ), + ) + } else { + podSpecOpts = createPodSpecOpts(*customMasterPodSpecOpts...) + } + masterPod := e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(podSpecOpts...)) // Create nfd-master service @@ -257,6 +264,7 @@ var _ = SIGDescribe("NFD master and worker", func() { cleanupNode(f.ClientSet) cleanupCRs(nfdClient, f.Namespace.Name) + customMasterPodSpecOpts = nil }) // @@ -264,7 +272,6 @@ var _ = SIGDescribe("NFD master and worker", func() { // Context("and a single worker pod with fake source enabled", func() { It("it should decorate the node with the fake feature labels", func() { - fakeFeatureLabels := map[string]string{ nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature1": "true", nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2": "true", @@ -771,6 +778,77 @@ core: Expect(err).NotTo(HaveOccurred()) }) }) + + Context("and check whether master config passed successfully or not", func() { + BeforeEach(func() { + customMasterPodSpecOpts = &[]testpod.SpecOption{ + testpod.SpecWithContainerImage(dockerImage()), + testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"), + testpod.SpecWithTolerations(testTolerations), + } + cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", ` +denyLabelNs: ["*.denied.ns","random.unwanted.ns"] +`) + _, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) + It("master configuration should take place", func() { + // deploy node feature object + if !useNodeFeatureApi { + Skip("NodeFeature API not enabled") + } + + nodes, err := getNonControlPlaneNodes(f.ClientSet) + Expect(err).NotTo(HaveOccurred()) + + targetNodeName := nodes[0].Name + Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found") + + // Apply Node Feature object + By("Create NodeFeature object") + nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(nfdClient, "nodefeature-3.yaml", f.Namespace.Name, targetNodeName) + Expect(err).NotTo(HaveOccurred()) + + // Verify that denied label was not added + By("Verifying that denied labels were not added") + expectedLabels := map[string]k8sLabels{ + targetNodeName: { + nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-4": "obj-4", + "custom.vendor.io/e2e-nodefeature-test-3": "vendor-ns", + }, + } + Expect(checkForNodeLabels( + f.ClientSet, + expectedLabels, + nodes, + )).NotTo(HaveOccurred()) + By("Deleting NodeFeature object") + err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(context.TODO(), nodeFeatures[0], metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // TODO: Find a better way to handle the timeout that happens to reflect the configmap changes + Skip("Testing the master dynamic configuration") + // Verify that config changes were applied + By("Updating the master config") + Expect(testutils.UpdateConfigMap(f.ClientSet, "nfd-master-conf", f.Namespace.Name, "nfd-master.conf", ` +denyLabelNs: [] +`)) + By("Verifying that denied labels were removed") + expectedLabels = map[string]k8sLabels{ + targetNodeName: { + nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-4": "obj-4", + "custom.vendor.io/e2e-nodefeature-test-3": "vendor-ns", + "random.denied.ns/e2e-nodefeature-test-1": "denied-ns", + "random.unwanted.ns/e2e-nodefeature-test-2": "unwanted-ns", + }, + } + Expect(checkForNodeLabels( + f.ClientSet, + expectedLabels, + nodes, + )).NotTo(HaveOccurred()) + }) + }) }) } diff --git a/test/e2e/utils/configmap.go b/test/e2e/utils/configmap.go index 192d11023..90885b798 100644 --- a/test/e2e/utils/configmap.go +++ b/test/e2e/utils/configmap.go @@ -17,8 +17,12 @@ limitations under the License. package utils import ( + "context" + "fmt" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" ) // CreateConfigMap is a helper for creating a simple ConfigMap object with one key. @@ -30,3 +34,17 @@ func NewConfigMap(name, key, data string) *corev1.ConfigMap { Data: map[string]string{key: data}, } } + +// UpdateConfigMap is a helper for updating a ConfigMap object. +func UpdateConfigMap(c clientset.Interface, name, namespace, key, data string) error { + cm, err := c.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("configmap %s is not found", name) + } + cm.Data[key] = data + _, err = c.CoreV1().ConfigMaps(namespace).Update(context.TODO(), cm, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("error while updating configmap with name %s", name) + } + return nil +}