test/e2e: add tests for NodeFeatureRules

Add tests covering the basic functionality of NodeFeatureRule objects,
covering different feature types ("flag features", "attribute features"
and "instance features") as well as backreferencing (using the output of
previously run rules) and templating. The test relies on the "fake"
feature source and its default configuration.

test/e2e/data/nodefeaturerule-1.yaml

@@ -0,0 +1,76 @@
+apiVersion: nfd.k8s-sigs.io/v1alpha1
+kind: NodeFeatureRule
+metadata:
+  name: e2e-test-1
+spec:
+  rules:
+    #
+    # Simple test rules for flag features
+    #
+    - name: "e2e-flag-test-1"
+      labels:
+        e2e-flag-test-1: "true"
+      vars:
+        e2e-flag-test-1.not: "false"
+      matchFeatures:
+        - feature: "fake.flag"
+          matchExpressions:
+            "flag_1": {op: Exists}
+
+    # Negative test not supposed to create a label
+    - name: "e2e-flag-test-neg-1"
+      labels:
+        e2e-flag-test-neg-1: "true"
+      matchFeatures:
+        - feature: "fake.flag"
+          matchExpressions:
+            "flag_1": {op: DoesNotExist}
+
+    #
+    # Simple test rules for attribute features
+    #
+    - name: "e2e-attribute-test-1"
+      labels:
+        e2e-attribute-test-1: "true"
+      vars:
+        e2e-attribute-test-1.not: "false"
+      matchFeatures:
+        - feature: "fake.attribute"
+          matchExpressions:
+            "attr_1": {op: IsTrue}
+            "attr_2": {op: IsFalse}
+
+    # Negative test not supposed to create a label
+    - name: "e2e-attribute-test-neg-1"
+      labels:
+        e2e-attribute-test-neg-1: "true"
+      matchFeatures:
+        - feature: "fake.attribute"
+          matchExpressions:
+            "attr_1": {op: IsTrue}
+            "attr_2": {op: IsTrue}
+
+    #
+    # Simple test rules for instnace features
+    #
+    - name: "e2e-instance-test-1"
+      labels:
+        e2e-instance-test-1: "true"
+      vars:
+        e2e-instance-test-1.not: "false"
+        e2e-instance-test-1.123: "123"
+      matchFeatures:
+        - feature: "fake.instance"
+          matchExpressions:
+            "attr_1": {op: In, value: ["true"]}
+            "attr_3": {op: Gt, value: ["10"]}
+
+    # Negative test not supposed to create a label
+    - name: "e2e-instance-test-neg-1"
+      labels:
+        e2e-instance-test-neg-1: "true"
+      matchFeatures:
+        - feature: "fake.instance"
+          matchExpressions:
+            "attr_1": {op: In, value: ["true"]}
+            "attr_3": {op: Lt, value: ["10"]}

test/e2e/data/nodefeaturerule-2.yaml

@@ -0,0 +1,41 @@
+apiVersion: nfd.k8s-sigs.io/v1alpha1
+kind: NodeFeatureRule
+metadata:
+  name: e2e-test-2
+spec:
+  rules:
+    #
+    # More complex rule testing backreferencing and matchAny field
+    #
+    - name: "e2e-matchany-test-1"
+      labels:
+        e2e-matchany-test-1: "true"
+      vars:
+        e2e-instance-test-1.not: "false"
+      matchFeatures:
+        - feature: "rule.matched"
+          matchExpressions:
+            "e2e-attribute-test-1": {op: InRegexp, value: ["^tru"]}
+            "e2e-instance-test-1.123": {op: In, value: ["1", "12", "123"]}
+      matchAny:
+        - matchFeatures:
+            - feature: "fake.instance"
+              matchExpressions:
+                "attr_1": {op: In, value: ["nomatch"]}
+        - matchFeatures:
+            - feature: "fake.instance"
+              matchExpressions:
+                "attr_3": {op: In, value: ["100"]}
+
+    #
+    # Simple test for templating
+    #
+    - name: "e2e-template-test-1"
+      labelsTemplate: |
+        {{ range .fake.instance }}e2e-template-test-1-{{ .name }}=found
+        {{ end }}
+      matchFeatures:
+        - feature: "fake.instance"
+          matchExpressions:
+            "attr_1": {op: In, value: ["true"]}
+

test/e2e/node_feature_discovery.go

@@ -24,10 +24,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	v1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	clientset "k8s.io/client-go/kubernetes"
@@ -35,6 +38,7 @@ import (
 	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
 	e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+	nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned"
 
 	nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1"
 	"sigs.k8s.io/node-feature-discovery/source/custom"
@@ -419,6 +423,103 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
 			})
 		})
 
-	})
+		//
+		// Test NodeFeatureRule
+		//
+		Context("and nfd-worker and NodeFeatureRules objects deployed", func() {
+			var extClient *extclient.Clientset
+			var nfdClient *nfdclient.Clientset
+			var crd *apiextensionsv1.CustomResourceDefinition
 
+			BeforeEach(func() {
+				// Create clients for apiextensions and our CRD api
+				extClient = extclient.NewForConfigOrDie(f.ClientConfig())
+				nfdClient = nfdclient.NewForConfigOrDie(f.ClientConfig())
+
+				// Create CRDs
+				By("Creating NodeFeatureRule CRD")
+				var err error
+				crd, err = testutils.CreateNodeFeatureRulesCRD(extClient)
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			AfterEach(func() {
+				err := extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{})
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It("custom labels from the NodeFeatureRule rules should be created", func() {
+				By("Creating nfd-worker daemonset")
+				workerArgs := []string{"-feature-sources=fake", "-label-sources=", "-sleep-interval=1s"}
+				workerDS := testutils.NFDWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), workerArgs)
+				workerDS, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{})
+				Expect(err).NotTo(HaveOccurred())
+
+				By("Waiting for daemonset pods to be ready")
+				Expect(testutils.WaitForPodsReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred())
+
+				expected := map[string]string{
+					"feature.node.kubernetes.io/e2e-flag-test-1":      "true",
+					"feature.node.kubernetes.io/e2e-attribute-test-1": "true",
+					"feature.node.kubernetes.io/e2e-instance-test-1":  "true"}
+
+				By("Creating NodeFeatureRules #1")
+				Expect(testutils.CreateNodeFeatureRuleFromFile(nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred())
+
+				By("Verifying node labels from NodeFeatureRules #1")
+				Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred())
+
+				By("Creating NodeFeatureRules #2")
+				Expect(testutils.CreateNodeFeatureRuleFromFile(nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred())
+
+				// Add features from NodeFeatureRule #2
+				expected["feature.node.kubernetes.io/e2e-matchany-test-1"] = "true"
+				expected["feature.node.kubernetes.io/e2e-template-test-1-instance_1"] = "found"
+				expected["feature.node.kubernetes.io/e2e-template-test-1-instance_2"] = "found"
+
+				By("Verifying node labels from NodeFeatureRules #1 and #2")
+				Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred())
+			})
+		})
+	})
 })
+
+// waitForNfdNodeLabels waits for node to be labeled as expected.
+func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) error {
+	poll := func() error {
+		nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+		if err != nil {
+			return err
+		}
+		for _, node := range nodeList.Items {
+			labels := nfdLabels(node.Labels)
+			if !cmp.Equal(expected, labels) {
+				return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(expected, labels))
+			}
+		}
+		return nil
+	}
+
+	// Simple and stupid re-try loop
+	var err error
+	for retry := 0; retry < 3; retry++ {
+		if err = poll(); err == nil {
+			return nil
+		}
+		time.Sleep(2 * time.Second)
+	}
+	return err
+}
+
+// nfdLabels gets labels that are in the nfd label namespace.
+func nfdLabels(labels map[string]string) map[string]string {
+	ret := map[string]string{}
+
+	for key, val := range labels {
+		if strings.HasPrefix(key, nfdv1alpha1.FeatureLabelNs) {
+			ret[key] = val
+		}
+	}
+	return ret
+
+}

test/e2e/utils/crd.go

@@ -0,0 +1,110 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apiruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/scheme"
+	nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1"
+	nfdclientset "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned"
+	nfdscheme "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned/scheme"
+)
+
+var packagePath string
+
+// CreateNodeFeatureRulesCRD creates the NodeFeatureRule CRD in the API server.
+func CreateNodeFeatureRulesCRD(cli extclient.Interface) (*apiextensionsv1.CustomResourceDefinition, error) {
+	crd, err := crdFromFile(filepath.Join(packagePath, "..", "..", "..", "deployment", "base", "nfd-crds", "nodefeaturerule-crd.yaml"))
+	if err != nil {
+		return nil, err
+	}
+
+	// Delete existing CRD (if any) with this we also get rid of stale objects
+	err = cli.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{})
+	if err != nil && !errors.IsNotFound(err) {
+		return nil, fmt.Errorf("failed to delete NodeFeatureRule CRD: %w", err)
+	}
+
+	return cli.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{})
+}
+
+// CreateNodeFeatureRuleFromFile creates a NodeFeatureRule object from a given file located under test data directory.
+func CreateNodeFeatureRuleFromFile(cli nfdclientset.Interface, filename string) error {
+	obj, err := nodeFeatureRuleFromFile(filepath.Join(packagePath, "..", "data", filename))
+	if err != nil {
+		return err
+	}
+	_, err = cli.NfdV1alpha1().NodeFeatureRules().Create(context.TODO(), obj, metav1.CreateOptions{})
+	return err
+}
+
+func apiObjFromFile(path string, decoder apiruntime.Decoder) (apiruntime.Object, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	obj, _, err := decoder.Decode(data, nil, nil)
+	return obj, err
+}
+
+// crdFromFile creates a CustomResourceDefinition API object from a file.
+func crdFromFile(path string) (*apiextensionsv1.CustomResourceDefinition, error) {
+	obj, err := apiObjFromFile(path, scheme.Codecs.UniversalDeserializer())
+	if err != nil {
+		return nil, err
+	}
+
+	crd, ok := obj.(*apiextensionsv1.CustomResourceDefinition)
+	if !ok {
+		return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path)
+	}
+
+	return crd, nil
+}
+
+func nodeFeatureRuleFromFile(path string) (*nfdv1alpha1.NodeFeatureRule, error) {
+	obj, err := apiObjFromFile(path, nfdscheme.Codecs.UniversalDeserializer())
+	if err != nil {
+		return nil, err
+	}
+
+	crd, ok := obj.(*nfdv1alpha1.NodeFeatureRule)
+	if !ok {
+		return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path)
+	}
+
+	return crd, nil
+}
+
+func init() {
+	_, thisFile, _, _ := runtime.Caller(0)
+	packagePath = filepath.Dir(thisFile)
+
+	// Register k8s scheme to be able to create CRDs
+	_ = apiextensionsv1.AddToScheme(scheme.Scheme)
+}

test/e2e/utils/rbac.go

@@ -128,6 +128,11 @@ func createClusterRoleMaster(cs clientset.Interface) (*rbacv1.ClusterRole, error
 				Resources: []string{"nodes"},
 				Verbs:     []string{"get", "patch", "update"},
 			},
+			{
+				APIGroups: []string{"nfd.k8s-sigs.io"},
+				Resources: []string{"nodefeaturerules"},
+				Verbs:     []string{"get", "list", "watch"},
+			},
 			{
 				APIGroups: []string{"topology.node.k8s.io"},
 				Resources: []string{"noderesourcetopologies"},