pkg/apis/nfd: add variables to rule spec and support backreferences

Support backreferencing of output values from previous rules. Enables
complex rule setups where custom features are further combined together
to form even more sophisticated higher level labels. The labels created
by preceding rules are available as a special 'rule.matched' feature
(for matchFeatures to use).

If referencing rules accross multiple configs/CRDs care must be taken
with the ordering. Processing order of rules in nfd-worker:

1. Static rules
2. Files from /etc/kubernetes/node-feature-discovery/custom.d/
   in alphabetical order. Subdirectories are processed by reading their
   files in alphabetical order.
3. Custom rules from main nfd-worker.conf

In nfd-master, NodeFeatureRule objects are processed in alphabetical
order (based on their metadata.name).

This patch also adds new 'vars' fields to the rule spec. Like 'labels',
it is a map of key-value pairs but no labels are generated from these.
The values specified in 'vars' are only added for backreferencing into
the 'rules.matched' feature. This may by desired in schemes where the
output of certain rules is only used as intermediate variables for other
rules and no labels out of these are wanted.

An example setup:

  - name: "kernel feature"
    labels:
      kernel-feature:
    matchFeatures:
      - feature: kernel.version
        matchExpressions:
          major: {op: Gt, value: ["4"]}

  - name: "intermediate var feature"
    vars:
      nolabel-feature: "true"
    matchFeatures:
      - feature: cpu.cpuid
        matchExpressions:
          AVX512F: {op: Exists}
      - feature: pci.device
        matchExpressions:
          vendor: {op: In, value: ["8086"]}
          device: {op: In, value: ["1234", "1235"]}

  - name: top-level-feature
    matchFeatures:
      - feature: rule.matched
        matchExpressions:
          kernel-feature: "true"
          nolabel-feature: "true"

deployment/base/nfd-crds/cr-sample.yaml

@@ -111,3 +111,29 @@ spec:
         - feature: cpu.cpuid
           matchExpressions:
             AVX: {op: Exists}
+
+    # The following examples demonstrate vars field and back-referencing
+    # previous labels and vars
+    - name: "my dummy kernel rule"
+      labels:
+        "my.kernel.feature": "true"
+      matchFeatures:
+        - feature: kernel.version
+          matchExpressions:
+            major: {op: Gt, value: ["2"]}
+
+    - name: "my dummy rule with no labels"
+      vars:
+        "my.dummy.var": "1"
+      matchFeatures:
+        - feature: cpu.cpuid
+          matchExpressions: {}
+
+    - name: "my rule using backrefs"
+      labels:
+        "my.backref.feature": "true"
+      matchFeatures:
+        - feature: rule.matched
+          matchExpressions:
+            my.kernel.feature: {op: IsTrue}
+            my.dummy.var: {op: Gt, value: ["0"]}

deployment/base/nfd-crds/nodefeaturerule-crd.yaml

@@ -188,6 +188,15 @@ spec:
                     name:
                       description: Name of the rule.
                       type: string
+                    vars:
+                      additionalProperties:
+                        type: string
+                      description: Vars is the variables to store if the rule matches.
+                        Variables do not directly inflict any changes in the node
+                        object. However, they can be referenced from other rules enabling
+                        more complex rule hierarchies, without exposing intermediary
+                        output values as labels.
+                      type: object
                   required:
                   - name
                   type: object

deployment/components/worker-config/nfd-worker.conf.example

@@ -226,3 +226,29 @@
 #        - feature: cpu.cpuid
 #          matchExpressions:
 #            AVX: {op: Exists}
+#
+#    # The following examples demonstrate vars field and back-referencing
+#    # previous labels and vars
+#    - name: "my dummy kernel rule"
+#      labels:
+#        "my.kernel.feature": "true"
+#      matchFeatures:
+#        - feature: kernel.version
+#          matchExpressions:
+#            major: {op: Gt, value: ["2"]}
+#
+#    - name: "my dummy rule with no labels"
+#      vars:
+#        "my.dummy.var": "1"
+#      matchFeatures:
+#        - feature: cpu.cpuid
+#          matchExpressions: {}
+#
+#    - name: "my rule using backrefs"
+#      labels:
+#        "my.backref.feature": "true"
+#      matchFeatures:
+#        - feature: rule.matched
+#          matchExpressions:
+#            my.kernel.feature: {op: IsTrue}
+#            my.dummy.var: {op: Gt, value: ["0"]}

deployment/helm/node-feature-discovery/manifests/nodefeaturerule-crd.yaml

@@ -188,6 +188,15 @@ spec:
                     name:
                       description: Name of the rule.
                       type: string
+                    vars:
+                      additionalProperties:
+                        type: string
+                      description: Vars is the variables to store if the rule matches.
+                        Variables do not directly inflict any changes in the node
+                        object. However, they can be referenced from other rules enabling
+                        more complex rule hierarchies, without exposing intermediary
+                        output values as labels.
+                      type: object
                   required:
                   - name
                   type: object

deployment/helm/node-feature-discovery/values.yaml

@@ -315,6 +315,32 @@ worker:
     #        - feature: cpu.cpuid
     #          matchExpressions:
     #            AVX: {op: Exists}
+    #
+    #    # The following examples demonstrate vars field and back-referencing
+    #    # previous labels and vars
+    #    - name: "my dummy kernel rule"
+    #      labels:
+    #        "my.kernel.feature": "true"
+    #      matchFeatures:
+    #        - feature: kernel.version
+    #          matchExpressions:
+    #            major: {op: Gt, value: ["2"]}
+    #
+    #    - name: "my dummy rule with no labels"
+    #      vars:
+    #        "my.dummy.var": "1"
+    #      matchFeatures:
+    #        - feature: cpu.cpuid
+    #          matchExpressions: {}
+    #
+    #    - name: "my rule using backrefs"
+    #      labels:
+    #        "my.backref.feature": "true"
+    #      matchFeatures:
+    #        - feature: rule.matched
+    #          matchExpressions:
+    #            my.kernel.feature: {op: IsTrue}
+    #            my.dummy.var: {op: Gt, value: ["0"]}
 ### <NFD-WORKER-CONF-END-DO-NOT-REMOVE>
 
   podSecurityContext: {}

pkg/api/feature/feature.go

@@ -50,3 +50,18 @@ func NewInstanceFeature(attrs map[string]string) *InstanceFeature {
 	}
 	return &InstanceFeature{Attributes: attrs}
 }
+
+// InsertFeatureValues inserts new values into a specific feature.
+func InsertFeatureValues(f Features, domain, feature string, values map[string]string) {
+	if _, ok := f[domain]; !ok {
+		f[domain] = NewDomainFeatures()
+	}
+	if _, ok := f[domain].Values[feature]; !ok {
+		f[domain].Values[feature] = NewValueFeatures(values)
+		return
+	}
+
+	for k, v := range values {
+		f[domain].Values[feature].Elements[k] = v
+	}
+}

pkg/apis/nfd/v1alpha1/rule.go

@@ -17,26 +17,35 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"bytes"
+	"fmt"
 	"strings"
 	"text/template"
 
-	"bytes"
+	"k8s.io/klog/v2"
-	"fmt"
 
 	"sigs.k8s.io/node-feature-discovery/pkg/api/feature"
 	"sigs.k8s.io/node-feature-discovery/pkg/utils"
 )
 
+// RuleOutput contains the output out rule execution.
+// +k8s:deepcopy-gen=false
+type RuleOutput struct {
+	Labels map[string]string
+	Vars   map[string]string
+}
+
 // Execute the rule against a set of input features.
-func (r *Rule) Execute(features map[string]*feature.DomainFeatures) (map[string]string, error) {
+func (r *Rule) Execute(features feature.Features) (RuleOutput, error) {
-	ret := make(map[string]string)
+	labels := make(map[string]string)
+	vars := make(map[string]string)
 
 	if len(r.MatchAny) > 0 {
 		// Logical OR over the matchAny matchers
 		matched := false
 		for _, matcher := range r.MatchAny {
 			if m, err := matcher.match(features); err != nil {
-				return nil, err
+				return RuleOutput{}, err
 			} else if m != nil {
 				matched = true
 				utils.KlogDump(4, "matches for matchAny "+r.Name, "  ", m)
@@ -46,33 +55,41 @@ func (r *Rule) Execute(features map[string]*feature.DomainFeatures) (map[string]
 					// produce the same labels)
 					break
 				}
-				if err := r.executeLabelsTemplate(m, ret); err != nil {
+				if err := r.executeLabelsTemplate(m, labels); err != nil {
-					return nil, err
+					return RuleOutput{}, err
 				}
 
 			}
 		}
 		if !matched {
-			return nil, nil
+			klog.V(2).Infof("rule %q did not match", r.Name)
+			return RuleOutput{}, nil
 		}
 	}
 
 	if len(r.MatchFeatures) > 0 {
 		if m, err := r.MatchFeatures.match(features); err != nil {
-			return nil, err
+			return RuleOutput{}, err
 		} else if m == nil {
-			return nil, nil
+			klog.V(2).Infof("rule %q did not match", r.Name)
+			return RuleOutput{}, nil
 		} else {
 			utils.KlogDump(4, "matches for matchFeatures "+r.Name, "  ", m)
-			if err := r.executeLabelsTemplate(m, ret); err != nil {
+			if err := r.executeLabelsTemplate(m, labels); err != nil {
-				return nil, err
+				return RuleOutput{}, err
 			}
 		}
 	}
 
 	for k, v := range r.Labels {
-		ret[k] = v
+		labels[k] = v
 	}
+	for k, v := range r.Vars {
+		vars[k] = v
+	}
+
+	ret := RuleOutput{Labels: labels, Vars: vars}
+	utils.KlogDump(2, fmt.Sprintf("rule %q matched with: ", r.Name), "  ", ret)
 
 	return ret, nil
 }

pkg/apis/nfd/v1alpha1/rule_test.go

@@ -28,6 +28,7 @@ func TestRule(t *testing.T) {
 	r1 := Rule{Labels: map[string]string{"label-1": "", "label-2": "true"}}
 	r2 := Rule{
 		Labels: map[string]string{"label-1": "label-val-1"},
+		Vars:   map[string]string{"var-1": "var-val-1"},
 		MatchFeatures: FeatureMatcher{
 			FeatureMatcherTerm{
 				Feature: "domain-1.kf-1",
@@ -41,7 +42,7 @@ func TestRule(t *testing.T) {
 	// Test totally empty features
 	m, err := r1.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r1.Labels, m, "empty matcher should have matched empty features")
+	assert.Equal(t, r1.Labels, m.Labels, "empty matcher should have matched empty features")
 
 	_, err = r2.Execute(f)
 	assert.Error(t, err, "matching agains a missing domain should have returned an error")
@@ -52,7 +53,8 @@ func TestRule(t *testing.T) {
 
 	m, err = r1.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r1.Labels, m, "empty matcher should have matched empty features")
+	assert.Equal(t, r1.Labels, m.Labels, "empty matcher should have matched empty features")
+	assert.Empty(t, r1.Vars, "vars should be empty")
 
 	_, err = r2.Execute(f)
 	assert.Error(t, err, "matching agains a missing feature type should have returned an error")
@@ -64,11 +66,11 @@ func TestRule(t *testing.T) {
 
 	m, err = r1.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r1.Labels, m, "empty matcher should have matched empty features")
+	assert.Equal(t, r1.Labels, m.Labels, "empty matcher should have matched empty features")
 
 	m, err = r2.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Nil(t, m, "unexpected match")
+	assert.Nil(t, m.Labels, "unexpected match")
 
 	// Test non-empty feature sets
 	d.Keys["kf-1"].Elements["key-x"] = feature.Nil{}
@@ -78,17 +80,18 @@ func TestRule(t *testing.T) {
 
 	m, err = r1.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r1.Labels, m, "empty matcher should have matched empty features")
+	assert.Equal(t, r1.Labels, m.Labels, "empty matcher should have matched empty features")
 
 	// Match "key" features
 	m, err = r2.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Nil(t, m, "keys should not have matched")
+	assert.Nil(t, m.Labels, "keys should not have matched")
 
 	d.Keys["kf-1"].Elements["key-1"] = feature.Nil{}
 	m, err = r2.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r2.Labels, m, "keys should have matched")
+	assert.Equal(t, r2.Labels, m.Labels, "keys should have matched")
+	assert.Equal(t, r2.Vars, m.Vars, "vars should be present")
 
 	// Match "value" features
 	r3 := Rule{
@@ -104,12 +107,12 @@ func TestRule(t *testing.T) {
 	}
 	m, err = r3.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Nil(t, m, "values should not have matched")
+	assert.Nil(t, m.Labels, "values should not have matched")
 
 	d.Values["vf-1"].Elements["key-1"] = "val-1"
 	m, err = r3.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r3.Labels, m, "values should have matched")
+	assert.Equal(t, r3.Labels, m.Labels, "values should have matched")
 
 	// Match "instance" features
 	r4 := Rule{
@@ -125,12 +128,12 @@ func TestRule(t *testing.T) {
 	}
 	m, err = r4.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Nil(t, m, "instances should not have matched")
+	assert.Nil(t, m.Labels, "instances should not have matched")
 
 	d.Instances["if-1"].Elements[0].Attributes["attr-1"] = "val-1"
 	m, err = r4.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r4.Labels, m, "instances should have matched")
+	assert.Equal(t, r4.Labels, m.Labels, "instances should have matched")
 
 	// Test multiple feature matchers
 	r5 := Rule{
@@ -152,12 +155,12 @@ func TestRule(t *testing.T) {
 	}
 	m, err = r5.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Nil(t, m, "instances should not have matched")
+	assert.Nil(t, m.Labels, "instances should not have matched")
 
 	r5.MatchFeatures[0].MatchExpressions.Expressions["key-1"] = MustCreateMatchExpression(MatchIn, "val-1")
 	m, err = r5.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r5.Labels, m, "instances should have matched")
+	assert.Equal(t, r5.Labels, m.Labels, "instances should have matched")
 
 	// Test MatchAny
 	r5.MatchAny = []MatchAnyElem{
@@ -174,7 +177,7 @@ func TestRule(t *testing.T) {
 	}
 	m, err = r5.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Nil(t, m, "instances should not have matched")
+	assert.Nil(t, m.Labels, "instances should not have matched")
 
 	r5.MatchAny = append(r5.MatchAny,
 		MatchAnyElem{
@@ -190,7 +193,7 @@ func TestRule(t *testing.T) {
 	r5.MatchFeatures[0].MatchExpressions.Expressions["key-1"] = MustCreateMatchExpression(MatchIn, "val-1")
 	m, err = r5.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r5.Labels, m, "instances should have matched")
+	assert.Equal(t, r5.Labels, m.Labels, "instances should have matched")
 }
 
 func TestTemplating(t *testing.T) {
@@ -297,7 +300,7 @@ label-2=
 
 	m, err := r1.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, expectedLabels, m, "instances should have matched")
+	assert.Equal(t, expectedLabels, m.Labels, "instances should have matched")
 
 	//
 	// Test error cases
@@ -316,7 +319,7 @@ label-2=
 	r2.LabelsTemplate = "foo=bar"
 	m, err = r2.Execute(f)
 	assert.Nil(t, err)
-	assert.Equal(t, map[string]string{"foo": "bar"}, m, "instances should have matched")
+	assert.Equal(t, map[string]string{"foo": "bar"}, m.Labels, "instances should have matched")
 
 	r2.labelsTemplate = nil
 	r2.LabelsTemplate = "foo"

pkg/apis/nfd/v1alpha1/types.go

@@ -65,6 +65,13 @@ type Rule struct {
 	// +optional
 	LabelsTemplate string `json:"labelsTemplate"`
 
+	// Vars is the variables to store if the rule matches. Variables do not
+	// directly inflict any changes in the node object. However, they can be
+	// referenced from other rules enabling more complex rule hierarchies,
+	// without exposing intermediary output values as labels.
+	// +optional
+	Vars map[string]string `json:"vars"`
+
 	// MatchFeatures specifies a set of matcher terms all of which must match.
 	// +optional
 	MatchFeatures FeatureMatcher `json:"matchFeatures"`
@@ -177,3 +184,12 @@ const (
 	// expression must not have any values.
 	MatchIsFalse MatchOp = "IsFalse"
 )
+
+const (
+	// RuleBackrefDomain is the special feature domain for backreferencing
+	// output of preceding rules.
+	RuleBackrefDomain = "rule"
+	// RuleBackrefFeature is the special feature name for backreferencing
+	// output of preceding rules.
+	RuleBackrefFeature = "matched"
+)

pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go

@@ -308,6 +308,13 @@ func (in *Rule) DeepCopyInto(out *Rule) {
 			(*out)[key] = val
 		}
 	}
+	if in.Vars != nil {
+		in, out := &in.Vars, &out.Vars
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.MatchFeatures != nil {
 		in, out := &in.MatchFeatures, &out.MatchFeatures
 		*out = make(FeatureMatcher, len(*in))

pkg/nfd-master/nfd-master.go

@@ -43,7 +43,9 @@ import (
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 
+	"sigs.k8s.io/node-feature-discovery/pkg/api/feature"
 	"sigs.k8s.io/node-feature-discovery/pkg/apihelper"
+	nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1"
 	pb "sigs.k8s.io/node-feature-discovery/pkg/labeler"
 	topologypb "sigs.k8s.io/node-feature-discovery/pkg/topologyupdater"
 	"sigs.k8s.io/node-feature-discovery/pkg/utils"
@@ -499,6 +501,10 @@ func (m *nfdMaster) crLabels(r *pb.SetLabelsRequest) map[string]string {
 
 	l := make(map[string]string)
 	ruleSpecs, err := m.nfdController.lister.List(labels.Everything())
+	sort.Slice(ruleSpecs, func(i, j int) bool {
+		return ruleSpecs[i].Name < ruleSpecs[j].Name
+	})
+
 	if err != nil {
 		klog.Errorf("failed to list LabelRule resources: %w", err)
 		return nil
@@ -519,10 +525,14 @@ func (m *nfdMaster) crLabels(r *pb.SetLabelsRequest) map[string]string {
 				klog.Errorf("failed to process Rule %q: %w", rule.Name, err)
 				continue
 			}
-			for k, v := range ruleOut {
+
+			for k, v := range ruleOut.Labels {
 				l[k] = v
 			}
-			utils.KlogDump(1, "", "  ", ruleOut)
+
+			// Feed back rule output to features map for subsequent rules to match
+			feature.InsertFeatureValues(r.Features, nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Labels)
+			feature.InsertFeatureValues(r.Features, nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Vars)
 		}
 	}
 

source/custom/custom.go

@@ -124,31 +124,34 @@ func (s *customSource) GetLabels() (source.FeatureLabels, error) {
 			continue
 		}
 
-		for n, v := range ruleOut {
+		for n, v := range ruleOut.Labels {
 			labels[n] = v
 		}
+		// Feed back rule output to features map for subsequent rules to match
+		feature.InsertFeatureValues(domainFeatures, nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Labels)
+		feature.InsertFeatureValues(domainFeatures, nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Vars)
 	}
 	return labels, nil
 }
 
-func (r *CustomRule) execute(features map[string]*feature.DomainFeatures) (map[string]string, error) {
+func (r *CustomRule) execute(features map[string]*feature.DomainFeatures) (nfdv1alpha1.RuleOutput, error) {
 	if r.LegacyRule != nil {
 		ruleOut, err := r.LegacyRule.execute(features)
 		if err != nil {
-			return nil, fmt.Errorf("failed to execute legacy rule %s: %w", r.LegacyRule.Name, err)
+			return nfdv1alpha1.RuleOutput{}, fmt.Errorf("failed to execute legacy rule %s: %w", r.LegacyRule.Name, err)
 		}
-		return ruleOut, err
+		return nfdv1alpha1.RuleOutput{Labels: ruleOut}, nil
 	}
 
 	if r.Rule != nil {
 		ruleOut, err := r.Rule.Execute(features)
 		if err != nil {
-			return nil, fmt.Errorf("failed to execute rule %s: %w", r.Rule.Name, err)
+			return ruleOut, fmt.Errorf("failed to execute rule %s: %w", r.Rule.Name, err)
 		}
-		return ruleOut, err
+		return ruleOut, nil
 	}
 
-	return nil, fmt.Errorf("BUG: an empty rule, this really should not happen")
+	return nfdv1alpha1.RuleOutput{}, fmt.Errorf("BUG: an empty rule, this really should not happen")
 }
 
 func (r *LegacyRule) execute(features map[string]*feature.DomainFeatures) (map[string]string, error) {