From a1a2429df1db2e99a0bfe3eb3b0f138ad0f92916 Mon Sep 17 00:00:00 2001 From: Adrian Chiris Date: Sun, 16 Feb 2020 16:27:03 +0200 Subject: [PATCH 1/7] Move PCI introspection logic to its own internal package This will enable code reuse across sources while preventing packages which are not under 'source' to import it. subsequent commits will introduce the 'custom' source which will use the logic. --- source/internal/pci_utils.go | 98 ++++++++++++++++++++++++++++++++++++ source/pci/pci.go | 63 ++--------------------- 2 files changed, 102 insertions(+), 59 deletions(-) create mode 100644 source/internal/pci_utils.go diff --git a/source/internal/pci_utils.go b/source/internal/pci_utils.go new file mode 100644 index 000000000..4caec5687 --- /dev/null +++ b/source/internal/pci_utils.go @@ -0,0 +1,98 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pciutils + +import ( + "fmt" + "io/ioutil" + "log" + "path" + "strings" +) + +type PciDeviceInfo map[string]string + +var DefaultPciDevAttrs = []string{"class", "vendor", "device", "subsystem_vendor", "subsystem_device"} +var ExtraPciDevAttrs = []string{"sriov_totalvfs"} + +// Read a single PCI device attribute +// A PCI attribute in this context, maps to the corresponding sysfs file +func readSingleAttribute(devPath string, attrName string) (string, error) { + data, err := ioutil.ReadFile(path.Join(devPath, attrName)) + if err != nil { + return "", fmt.Errorf("failed to read device attribute %s: %v", attrName, err) + } + // Strip whitespace and '0x' prefix + attrVal := strings.TrimSpace(strings.TrimPrefix(string(data), "0x")) + + if attrName == "class" && len(attrVal) > 4 { + // Take four first characters, so that the programming + // interface identifier gets stripped from the raw class code + attrVal = attrVal[0:4] + } + return attrVal, nil +} + +// Read information of one PCI device +func readDevInfo(devPath string, deviceAttrSpec map[string]bool) (PciDeviceInfo, error) { + info := PciDeviceInfo{} + + for attr, must := range deviceAttrSpec { + attrVal, err := readSingleAttribute(devPath, attr) + if err != nil { + if must { + return info, fmt.Errorf("Failed to read device %s: %s", attr, err) + } else { + continue + } + + } + info[attr] = attrVal + } + return info, nil +} + +// List available PCI devices and retrieve device attributes. +// deviceAttrSpec is a map which specifies which attributes to retrieve. +// a false value for a specific attribute marks the attribute as optional. +// a true value for a specific attribute marks the attribute as mandatory. +// "class" attribute is considered mandatory. +// DetectPci() will fail if the retrieval of a mandatory attribute fails. +func DetectPci(deviceAttrSpec map[string]bool) (map[string][]PciDeviceInfo, error) { + const basePath = "/sys/bus/pci/devices/" + devInfo := make(map[string][]PciDeviceInfo) + + devices, err := ioutil.ReadDir(basePath) + if err != nil { + return nil, err + } + // "class" is a mandatory attribute, inject it to spec if needed. + deviceAttrSpec["class"] = true + + // Iterate over devices + for _, device := range devices { + info, err := readDevInfo(path.Join(basePath, device.Name()), deviceAttrSpec) + if err != nil { + log.Print(err) + continue + } + class := info["class"] + devInfo[class] = append(devInfo[class], info) + } + + return devInfo, nil +} diff --git a/source/pci/pci.go b/source/pci/pci.go index ed70f95a7..f1c337cbb 100644 --- a/source/pci/pci.go +++ b/source/pci/pci.go @@ -18,16 +18,13 @@ package pci import ( "fmt" - "io/ioutil" "log" - "path" "strings" "sigs.k8s.io/node-feature-discovery/source" + "sigs.k8s.io/node-feature-discovery/source/internal" ) -type pciDeviceInfo map[string]string - type NFDConfig struct { DeviceClassWhitelist []string `json:"deviceClassWhitelist,omitempty"` DeviceLabelFields []string `json:"deviceLabelFields,omitempty"` @@ -38,9 +35,6 @@ var Config = NFDConfig{ DeviceLabelFields: []string{"class", "vendor"}, } -var devLabelAttrs = []string{"class", "vendor", "device", "subsystem_vendor", "subsystem_device"} -var extraDevAttrs = []string{"sriov_totalvfs"} - // Implement FeatureSource interface type Source struct{} @@ -58,7 +52,7 @@ func (s Source) Discover() (source.Features, error) { configLabelFields[field] = true } - for _, attr := range devLabelAttrs { + for _, attr := range pciutils.DefaultPciDevAttrs { if _, ok := configLabelFields[attr]; ok { deviceLabelFields = append(deviceLabelFields, attr) delete(configLabelFields, attr) @@ -79,14 +73,14 @@ func (s Source) Discover() (source.Features, error) { // Read extraDevAttrs + configured or default labels. Attributes // set to 'true' are considered must-have. deviceAttrs := map[string]bool{} - for _, label := range extraDevAttrs { + for _, label := range pciutils.ExtraPciDevAttrs { deviceAttrs[label] = false } for _, label := range deviceLabelFields { deviceAttrs[label] = true } - devs, err := detectPci(deviceAttrs) + devs, err := pciutils.DetectPci(deviceAttrs) if err != nil { return nil, fmt.Errorf("Failed to detect PCI devices: %s", err.Error()) } @@ -112,54 +106,5 @@ func (s Source) Discover() (source.Features, error) { } } } - return features, nil } - -// Read information of one PCI device -func readDevInfo(devPath string, devAttrs map[string]bool) (pciDeviceInfo, error) { - info := pciDeviceInfo{} - for attr, must := range devAttrs { - data, err := ioutil.ReadFile(path.Join(devPath, attr)) - if err != nil { - if must { - return info, fmt.Errorf("Failed to read device %s: %s", attr, err) - } else { - continue - } - } - // Strip whitespace and '0x' prefix - info[attr] = strings.TrimSpace(strings.TrimPrefix(string(data), "0x")) - - if attr == "class" && len(info[attr]) > 4 { - // Take four first characters, so that the programming - // interface identifier gets stripped from the raw class code - info[attr] = info[attr][0:4] - } - } - return info, nil -} - -// List available PCI devices -func detectPci(devAttrs map[string]bool) (map[string][]pciDeviceInfo, error) { - const basePath = "/sys/bus/pci/devices/" - devInfo := make(map[string][]pciDeviceInfo) - - devices, err := ioutil.ReadDir(basePath) - if err != nil { - return nil, err - } - - // Iterate over devices - for _, device := range devices { - info, err := readDevInfo(path.Join(basePath, device.Name()), devAttrs) - if err != nil { - log.Print(err) - continue - } - class := info["class"] - devInfo[class] = append(devInfo[class], info) - } - - return devInfo, nil -} From 0cfe03012b1fdd738c774939efeaad362b2154aa Mon Sep 17 00:00:00 2001 From: Adrian Chiris Date: Sun, 16 Feb 2020 17:22:07 +0200 Subject: [PATCH 2/7] Add new feature Source - Custom with stubbed implementation --- source/custom/custom.go | 47 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 source/custom/custom.go diff --git a/source/custom/custom.go b/source/custom/custom.go new file mode 100644 index 000000000..bae3f15b7 --- /dev/null +++ b/source/custom/custom.go @@ -0,0 +1,47 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package custom + +import ( + "encoding/json" + + "sigs.k8s.io/node-feature-discovery/source" +) + +type MatchRule struct { +} + +type CustomFeature struct { + Name string `json:"name"` + MatchOn []MatchRule `json:"matchOn"` +} + +type NFDConfig []CustomFeature + +var Config = NFDConfig{} + +// Implements FeatureSource Interface +type Source struct{} + +// Return name of the feature source +func (s Source) Name() string { return "custom" } + +// Discover features +func (s Source) Discover() (source.Features, error) { + features := source.Features{} + return features, nil +} From b9ab93559b1f6799f978933b8a2ab65e7183cb9b Mon Sep 17 00:00:00 2001 From: Adrian Chiris Date: Sun, 16 Feb 2020 17:24:06 +0200 Subject: [PATCH 3/7] Add Match Rules package to be used in Custom Source - Add a Rule interface to help describe the contract between a match rule and the Custom source that uses it. - Add PciIdRule - a rule that matches on the PCI attributes: class, vendor, device. Each is provided as a list of elements(strings). Match operation: OR will be performed per element and AND will be performed per attribute. An empty attribute will not be included in the matching process. Example: { "class": ["0200"] "vendor": ["15b3"] "device": ["1014", "1016"] } - Add LoadedKmodRule - a rule that matches a list of kernel modules with the kernel modules currently loaded in the node. Example: { ["rdma_cm", "ib_core"] } --- source/custom/custom.go | 3 +- source/custom/rules/loaded_kmod_rule.go | 60 +++++++++++++++++ source/custom/rules/pci_id_rule.go | 87 +++++++++++++++++++++++++ source/custom/rules/rule.go | 22 +++++++ 4 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 source/custom/rules/loaded_kmod_rule.go create mode 100644 source/custom/rules/pci_id_rule.go create mode 100644 source/custom/rules/rule.go diff --git a/source/custom/custom.go b/source/custom/custom.go index bae3f15b7..8497f2aed 100644 --- a/source/custom/custom.go +++ b/source/custom/custom.go @@ -17,11 +17,10 @@ limitations under the License. package custom import ( - "encoding/json" - "sigs.k8s.io/node-feature-discovery/source" ) +// Custom Features Configurations type MatchRule struct { } diff --git a/source/custom/rules/loaded_kmod_rule.go b/source/custom/rules/loaded_kmod_rule.go new file mode 100644 index 000000000..02bcc39be --- /dev/null +++ b/source/custom/rules/loaded_kmod_rule.go @@ -0,0 +1,60 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rules + +import ( + "fmt" + "io/ioutil" + "strings" +) + +// Rule that matches on loaded kernel modules in the system +type LoadedKModRule []string + +const kmodProcfsPath = "/proc/modules" + +func (kmods *LoadedKModRule) Match() (bool, error) { + loadedModules, err := kmods.getLoadedModules() + if err != nil { + return false, fmt.Errorf("failed to get loaded kernel modules. %s", err.Error()) + } + for _, kmod := range *kmods { + if _, ok := loadedModules[kmod]; !ok { + // kernel module not loaded + return false, nil + } + } + return true, nil +} + +func (kmods *LoadedKModRule) getLoadedModules() (map[string]struct{}, error) { + out, err := ioutil.ReadFile(kmodProcfsPath) + if err != nil { + return nil, fmt.Errorf("failed to read file %s: %s", kmodProcfsPath, err.Error()) + } + + loadedMods := make(map[string]struct{}) + for _, line := range strings.Split(string(out), "\n") { + // skip empty lines + if len(line) == 0 { + continue + } + // append loaded module + loadedMods[strings.Fields(line)[0]] = struct{}{} + } + return loadedMods, nil +} diff --git a/source/custom/rules/pci_id_rule.go b/source/custom/rules/pci_id_rule.go new file mode 100644 index 000000000..ff6cf62f3 --- /dev/null +++ b/source/custom/rules/pci_id_rule.go @@ -0,0 +1,87 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rules + +import ( + "fmt" + pciutils "sigs.k8s.io/node-feature-discovery/source/internal" +) + +// Rule that matches on the following PCI device attributes: +// each device attribute will be a list elements(strings). +// Match operation: OR will be performed per element and AND will be performed per attribute. +// An empty attribute will not be included in the matching process. +type PciIdRuleInput struct { + Class []string `json:"class,omitempty"` + Vendor []string `json:"vendor,omitempty"` + Device []string `json:"device,omitempty"` +} + +type PciIdRule struct { + PciIdRuleInput +} + +// Match PCI devices on the provided list of PCI regular expressions +func (r *PciIdRule) Match() (bool, error) { + devAttr := map[string]bool{} + for _, attr := range []string{"class", "vendor", "device"} { + devAttr[attr] = true + } + allDevs, err := pciutils.DetectPci(devAttr) + if err != nil { + return false, fmt.Errorf("failed to detect PCI devices: %s", err.Error()) + } + + for _, classDevs := range allDevs { + for _, dev := range classDevs { + // match rule on a single device + if r.matchDevOnRule(dev) { + return true, nil + } + } + } + return false, nil +} + +func (r *PciIdRule) matchDevOnRule(dev pciutils.PciDeviceInfo) bool { + if len(r.Class) == 0 && len(r.Vendor) == 0 && len(r.Device) == 0 { + return false + } + + if len(r.Class) > 0 && !in(dev["class"], r.Class) { + return false + } + + if len(r.Vendor) > 0 && !in(dev["vendor"], r.Vendor) { + return false + } + + if len(r.Device) > 0 && !in(dev["device"], r.Device) { + return false + } + + return true +} + +func in(item string, arr []string) bool { + for _, val := range arr { + if val == item { + return true + } + } + return false +} diff --git a/source/custom/rules/rule.go b/source/custom/rules/rule.go new file mode 100644 index 000000000..5632fccd0 --- /dev/null +++ b/source/custom/rules/rule.go @@ -0,0 +1,22 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rules + +type Rule interface { + // Match on rule + Match() (bool, error) +} From e4e3a9f68e533342d4574017c4aaef5b22e1b112 Mon Sep 17 00:00:00 2001 From: Adrian Chiris Date: Sun, 16 Feb 2020 17:33:27 +0200 Subject: [PATCH 4/7] Implement the 'custom' feature Source - Implement the 'custom' feature source utilizing the match rules implemented in previous commit. - Add a static custom feature list for: 1. rdma.capable - marks a node where devices that support RDMA are present. 2. rdma.enabled - marks a node where rdma modules have been loaded. A user may extend these features with additional match rules via NFD configuration file. --- source/custom/custom.go | 47 +++++++++++++++++++++++++ source/custom/rules/loaded_kmod_rule.go | 1 + source/custom/rules/pci_id_rule.go | 2 +- source/custom/static_features.go | 46 ++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 source/custom/static_features.go diff --git a/source/custom/custom.go b/source/custom/custom.go index 8497f2aed..300f79a47 100644 --- a/source/custom/custom.go +++ b/source/custom/custom.go @@ -17,11 +17,17 @@ limitations under the License. package custom import ( + "fmt" + "log" + "sigs.k8s.io/node-feature-discovery/source" + "sigs.k8s.io/node-feature-discovery/source/custom/rules" ) // Custom Features Configurations type MatchRule struct { + PciId *rules.PciIdRule `json:"pciId,omitempty""` + LoadedKMod *rules.LoadedKModRule `json:"loadedKMod,omitempty""` } type CustomFeature struct { @@ -42,5 +48,46 @@ func (s Source) Name() string { return "custom" } // Discover features func (s Source) Discover() (source.Features, error) { features := source.Features{} + allFeatureConfig := append(getStaticFeatureConfig(), Config...) + log.Printf("INFO: Custom features: %+v", allFeatureConfig) + // Iterate over features + for _, customFeature := range allFeatureConfig { + featureExist, err := s.discoverFeature(customFeature) + if err != nil { + return features, fmt.Errorf("failed to discover feature: %s. %s", customFeature.Name, err.Error()) + } + if featureExist { + features[customFeature.Name] = true + } + } return features, nil } + +// Process a single feature by Matching on the defined rules. +// A feature is present if all defined Rules in a MatchRule return a match. +func (s Source) discoverFeature(feature CustomFeature) (bool, error) { + for _, rule := range feature.MatchOn { + // PCI ID rule + if rule.PciId != nil { + match, err := rule.PciId.Match() + if err != nil { + return false, err + } + if !match { + continue + } + } + // Loaded kernel module rule + if rule.LoadedKMod != nil { + match, err := rule.LoadedKMod.Match() + if err != nil { + return false, err + } + if !match { + continue + } + } + return true, nil + } + return false, nil +} diff --git a/source/custom/rules/loaded_kmod_rule.go b/source/custom/rules/loaded_kmod_rule.go index 02bcc39be..2bfb8c32f 100644 --- a/source/custom/rules/loaded_kmod_rule.go +++ b/source/custom/rules/loaded_kmod_rule.go @@ -27,6 +27,7 @@ type LoadedKModRule []string const kmodProcfsPath = "/proc/modules" +// Match loaded kernel modules on provided list of kernel modules func (kmods *LoadedKModRule) Match() (bool, error) { loadedModules, err := kmods.getLoadedModules() if err != nil { diff --git a/source/custom/rules/pci_id_rule.go b/source/custom/rules/pci_id_rule.go index ff6cf62f3..45931cddd 100644 --- a/source/custom/rules/pci_id_rule.go +++ b/source/custom/rules/pci_id_rule.go @@ -35,7 +35,7 @@ type PciIdRule struct { PciIdRuleInput } -// Match PCI devices on the provided list of PCI regular expressions +// Match PCI devices on provided PCI device attributes func (r *PciIdRule) Match() (bool, error) { devAttr := map[string]bool{} for _, attr := range []string{"class", "vendor", "device"} { diff --git a/source/custom/static_features.go b/source/custom/static_features.go new file mode 100644 index 000000000..8796380af --- /dev/null +++ b/source/custom/static_features.go @@ -0,0 +1,46 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package custom + +import ( + "sigs.k8s.io/node-feature-discovery/source/custom/rules" +) + +// getStaticFeatures returns statically configured custom features to discover +// e.g RMDA related features. NFD configuration file may extend these custom features by adding rules. +func getStaticFeatureConfig() []CustomFeature { + return []CustomFeature{ + CustomFeature{ + Name: "rdma.capable", + MatchOn: []MatchRule{ + MatchRule{ + PciId: &rules.PciIdRule{ + rules.PciIdRuleInput{Vendor: []string{"15b3"}}, + }, + }, + }, + }, + CustomFeature{ + Name: "rdma.available", + MatchOn: []MatchRule{ + MatchRule{ + LoadedKMod: &rules.LoadedKModRule{"ib_uverbs", "rdma_ucm"}, + }, + }, + }, + } +} From 192b3d7bdd6be38d255303dadaa439a82050ed29 Mon Sep 17 00:00:00 2001 From: Adrian Chiris Date: Sun, 16 Feb 2020 17:43:12 +0200 Subject: [PATCH 5/7] Add 'custom' feature Source to nfd-worker --- cmd/nfd-worker/main.go | 2 +- cmd/nfd-worker/main_test.go | 6 ++++-- pkg/nfd-worker/nfd-worker.go | 4 ++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cmd/nfd-worker/main.go b/cmd/nfd-worker/main.go index b11b2f4c9..48a8ac3a7 100644 --- a/cmd/nfd-worker/main.go +++ b/cmd/nfd-worker/main.go @@ -91,7 +91,7 @@ func argsParse(argv []string) (worker.Args, error) { in testing [Default: ] --sources= Comma separated list of feature sources. - [Default: cpu,iommu,kernel,local,memory,network,pci,storage,system] + [Default: cpu,custom,iommu,kernel,local,memory,network,pci,storage,system] --no-publish Do not publish discovered features to the cluster-local Kubernetes API server. --label-whitelist= Regular expression to filter label names to diff --git a/cmd/nfd-worker/main_test.go b/cmd/nfd-worker/main_test.go index 05824fa8d..90b51727c 100644 --- a/cmd/nfd-worker/main_test.go +++ b/cmd/nfd-worker/main_test.go @@ -23,6 +23,8 @@ import ( . "github.com/smartystreets/goconvey/convey" ) +var allSources = []string{"cpu", "custom", "iommu", "kernel", "local", "memory", "network", "pci", "storage", "system"} + func TestArgsParse(t *testing.T) { Convey("When parsing command line arguments", t, func() { Convey("When --no-publish and --oneshot flags are passed", func() { @@ -32,7 +34,7 @@ func TestArgsParse(t *testing.T) { So(args.SleepInterval, ShouldEqual, 60*time.Second) So(args.NoPublish, ShouldBeTrue) So(args.Oneshot, ShouldBeTrue) - So(args.Sources, ShouldResemble, []string{"cpu", "iommu", "kernel", "local", "memory", "network", "pci", "storage", "system"}) + So(args.Sources, ShouldResemble, allSources) So(len(args.LabelWhiteList), ShouldEqual, 0) So(err, ShouldBeNil) }) @@ -56,7 +58,7 @@ func TestArgsParse(t *testing.T) { Convey("args.labelWhiteList is set to appropriate value and args.sources is set to default value", func() { So(args.NoPublish, ShouldBeFalse) - So(args.Sources, ShouldResemble, []string{"cpu", "iommu", "kernel", "local", "memory", "network", "pci", "storage", "system"}) + So(args.Sources, ShouldResemble, allSources) So(args.LabelWhiteList, ShouldResemble, ".*rdt.*") So(err, ShouldBeNil) }) diff --git a/pkg/nfd-worker/nfd-worker.go b/pkg/nfd-worker/nfd-worker.go index e46e1421e..a353f8585 100644 --- a/pkg/nfd-worker/nfd-worker.go +++ b/pkg/nfd-worker/nfd-worker.go @@ -36,6 +36,7 @@ import ( "sigs.k8s.io/node-feature-discovery/pkg/version" "sigs.k8s.io/node-feature-discovery/source" "sigs.k8s.io/node-feature-discovery/source/cpu" + "sigs.k8s.io/node-feature-discovery/source/custom" "sigs.k8s.io/node-feature-discovery/source/fake" "sigs.k8s.io/node-feature-discovery/source/iommu" "sigs.k8s.io/node-feature-discovery/source/kernel" @@ -61,6 +62,7 @@ type NFDConfig struct { Cpu *cpu.NFDConfig `json:"cpu,omitempty"` Kernel *kernel.NFDConfig `json:"kernel,omitempty"` Pci *pci.NFDConfig `json:"pci,omitempty"` + Custom *custom.NFDConfig `json:"custom,omitempty"` } `json:"sources,omitempty"` } @@ -234,6 +236,7 @@ func configParse(filepath string, overrides string) error { config.Sources.Cpu = &cpu.Config config.Sources.Kernel = &kernel.Config config.Sources.Pci = &pci.Config + config.Sources.Custom = &custom.Config data, err := ioutil.ReadFile(filepath) if err != nil { @@ -276,6 +279,7 @@ func configureParameters(sourcesWhiteList []string, labelWhiteListStr string) (e pci.Source{}, storage.Source{}, system.Source{}, + custom.Source{}, // local needs to be the last source so that it is able to override // labels from other sources local.Source{}, From f5307ddf7acaea79d63039bbe7f89c1a71bb258c Mon Sep 17 00:00:00 2001 From: Adrian Chiris Date: Sun, 16 Feb 2020 18:36:38 +0200 Subject: [PATCH 6/7] Documentation updates - Update README.md with `custom` source - Update nfd-worker configuration example --- README.md | 83 +++++++++++++++++++++++++++++++++++++++++ nfd-worker.conf.example | 19 ++++++++++ 2 files changed, 102 insertions(+) diff --git a/README.md b/README.md index cd4789b13..f60efcc5f 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,7 @@ for up-to-date information about the required volume mounts. The current set of feature sources are the following: - CPU +- Custom - IOMMU - Kernel - Memory @@ -181,6 +182,7 @@ feature logically has sub-hierarchy, e.g. `sriov.capable` and ```json { "feature.node.kubernetes.io/cpu-": "true", + "feature.node.kubernetes.io/custom-": "true", "feature.node.kubernetes.io/iommu-": "true", "feature.node.kubernetes.io/kernel-": "", "feature.node.kubernetes.io/memory-": "true", @@ -251,6 +253,87 @@ capability might be supported but not enabled. | JSCVT | Perform Conversion to Match Javascript | DCPOP | Persistent Memory Support +### Custom Features +The Custom feature source allows the user to define features based on a mix of predefined rules. + +#### Rule Types + +##### PciId Rule +The PciId rule allows matching the PCI devices in the system on the following PCI attributes: `class`,`vendor` and +`device` identifiers. + +```yaml +pciId : + class: [, ...] + vendor: [, ...] + device: [, ...] +``` +Matching is done by performing a logical OR for elements of an attribute and logical AND between specified attributes for +each PCI device in the system. +At least one attribute must be specificed, missing attributes will not partake in the matching process. + +##### LoadedKMod Rule +The LoadedKMod rule allows matching the loaded kernel modules in the system against a provided list of kernel modules. + +```yaml +loadedKMod : [, ...] +``` + Matching is done by performing a logical AND for each provided kernel module, i.e the rule will match if all provided kernel modules are loaded + in the system. + +#### Multiple Rules Matching process +Specifying Rules to match on a feature is done by a list of rule elements. +Each rule element contains one or more rule types. + +Logical OR is performed between rule elements and logical AND is performed between rule types +of a given rule element. + +##### Example +```yaml +custom: + - name: "my.kernel.feature" + matchOn: + - loadedKMod: ["kmod1", "kmod2"] + - name: "my.pci.feature" + matchOn: + - pciId: + vendor: ["15b3"] + device: ["1014", "1017"] + - name: "my.combined.feature" + matchOn: + - loadedKMod : ["vendor_kmod1", "vendor_kmod2"] + pciId: + vendor: ["15b3"] + device: ["1014", "1017"] + - name: "my.accumulated.feature" + matchOn: + - loadedKMod : ["some_kmod1", "some_kmod2"] + - pciId: + vendor: ["15b3"] + device: ["1014", "1017"] +``` + +__In the example above:__ +- A node would contain the label: `feature.node.kubernetes.io/custom-my.kernel.feature=true` +if the node has `kmod1` _AND_ `kmod2` kernel modules loaded. +- A node would contain the label: `feature.node.kubernetes.io/custom-my.pci.feature=true` +if the node contains a PCI device with a PCI vendor ID of `15b3` _AND_ PCI device ID of `1014` _OR_ `1017`. +- A node would contain the label: `feature.node.kubernetes.io/custom-my.combined.feature=true` +if `vendor_kmod1` _AND_ `vendor_kmod2` kernel modules are loaded __AND__ the node contains a PCI device +with a PCI vendor ID of `15b3` _AND_ PCI device ID of `1014` _or_ `1017`. +- A node would contain the label: `feature.node.kubernetes.io/custom-my.accumulated.feature=true` +if `some_kmod1` _AND_ `some_kmod2` kernel modules are loaded __OR__ the node contains a PCI device +with a PCI vendor ID of `15b3` _AND_ PCI device ID of `1014` _OR_ `1017`. + +#### Statically defined features +Some feature labels which are common and generic are defined statically in the `custom` feature source. +A user may add additional rules to these feature labels by defining them in the `nfd-worker` configuration file + +| Feature | Attribute | Description | +| ------- | --------- | -----------| +| rdma | capable | The node has an RDMA capable Network adapter | +| rdma | enabled | The node has the needed RDMA modules loaded to run RDMA traffic | + ### IOMMU Features | Feature name | Description | diff --git a/nfd-worker.conf.example b/nfd-worker.conf.example index f52acb342..591385874 100644 --- a/nfd-worker.conf.example +++ b/nfd-worker.conf.example @@ -44,3 +44,22 @@ # - "device" # - "subsystem_vendor" # - "subsystem_device" +# custom: +# - name: "my.kernel.feature" +# matchOn: +# - loadedKMod: ["example_kmod1", "example_kmod2"] +# - name: "my.pci.feature" +# matchOn: +# - pciId: +# class: ["0200"] +# vendor: ["15b3"] +# device: ["1014", "1017"] +# - pciId : +# vendor: ["8086"] +# device: ["1000", "1100"] +# - name: "my.combined.feature" +# matchOn: +# - pciId: +# vendor: ["15b3"] +# device: ["1014", "1017"] +# loadedKMod : ["vendor_kmod1", "vendor_kmod2"] From 297f8134deb05dfabcdf234ba2f1c59a0525e3d7 Mon Sep 17 00:00:00 2001 From: Adrian Chiris Date: Sun, 22 Mar 2020 14:46:16 +0200 Subject: [PATCH 7/7] Clarify Custom Features section in README - Add Nomenclature - Rewording to match the defined nomenclature - Reorganize Custom Feature section for clarity --- README.md | 72 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f60efcc5f..e755fa0f6 100644 --- a/README.md +++ b/README.md @@ -255,40 +255,80 @@ capability might be supported but not enabled. ### Custom Features The Custom feature source allows the user to define features based on a mix of predefined rules. +A rule is provided input witch affects its process of matching for a defined feature. -#### Rule Types +To aid in making Custom Features clearer, we define a general and a per rule nomenclature, keeping things as +consistent as possible. +#### General Nomenclature & Definitions +``` +Rule :Represents a matching logic that is used to match on a feature. +Rule Input :The input a Rule is provided. This determines how a Rule performs the match operation. +Matcher :A composition of Rules, each Matcher may be composed of at most one instance of each Rule. +``` + +#### Custom Features Format (using the Nomenclature defined above) +```yaml +- name: + matchOn: + - : + [: ] + - + - ... + - ... + - +- +- ... +- ... +- +``` + +#### Matching process +Specifying Rules to match on a feature is done by providing a list of Matchers. +Each Matcher contains one or more Rules. + +Logical _OR_ is performed between Matchers and logical _AND_ is performed between Rules +of a given Matcher. + +#### Rules ##### PciId Rule -The PciId rule allows matching the PCI devices in the system on the following PCI attributes: `class`,`vendor` and -`device` identifiers. +###### Nomenclature +``` +Attribute :A PCI attribute. +Element :An identifier of the PCI attribute. +``` +The PciId Rule allows matching the PCI devices in the system on the following Attributes: `class`,`vendor` and +`device`. A list of Elements is provided for each Attribute. + +###### Format ```yaml pciId : class: [, ...] vendor: [, ...] device: [, ...] ``` -Matching is done by performing a logical OR for elements of an attribute and logical AND between specified attributes for + +Matching is done by performing a logical _OR_ between Elements of an Attribute and logical _AND_ between the specified Attributes for each PCI device in the system. -At least one attribute must be specificed, missing attributes will not partake in the matching process. +At least one Attribute must be specified. Missing attributes will not partake in the matching process. ##### LoadedKMod Rule -The LoadedKMod rule allows matching the loaded kernel modules in the system against a provided list of kernel modules. +###### Nomenclature +``` +Element :A kernel module +``` +The LoadedKMod Rule allows matching the loaded kernel modules in the system against a provided list of Elements. + +###### Format ```yaml loadedKMod : [, ...] ``` - Matching is done by performing a logical AND for each provided kernel module, i.e the rule will match if all provided kernel modules are loaded + Matching is done by performing logical _AND_ for each provided Element, i.e the Rule will match if all provided Elements (kernel modules) are loaded in the system. -#### Multiple Rules Matching process -Specifying Rules to match on a feature is done by a list of rule elements. -Each rule element contains one or more rule types. - -Logical OR is performed between rule elements and logical AND is performed between rule types -of a given rule element. - -##### Example +#### Example ```yaml custom: - name: "my.kernel.feature" @@ -327,7 +367,7 @@ with a PCI vendor ID of `15b3` _AND_ PCI device ID of `1014` _OR_ `1017`. #### Statically defined features Some feature labels which are common and generic are defined statically in the `custom` feature source. -A user may add additional rules to these feature labels by defining them in the `nfd-worker` configuration file +A user may add additional Matchers to these feature labels by defining them in the `nfd-worker` configuration file. | Feature | Attribute | Description | | ------- | --------- | -----------|