Merge pull request #848 from marquiz/devel/e2e-nodefeaturerule

test/e2e: add tests for NodeFeatureRules

test/e2e/data/nodefeaturerule-1.yaml

@@ -0,0 +1,76 @@
+apiVersion: nfd.k8s-sigs.io/v1alpha1
+kind: NodeFeatureRule
+metadata:
+  name: e2e-test-1
+spec:
+  rules:
+    #
+    # Simple test rules for flag features
+    #
+    - name: "e2e-flag-test-1"
+      labels:
+        e2e-flag-test-1: "true"
+      vars:
+        e2e-flag-test-1.not: "false"
+      matchFeatures:
+        - feature: "fake.flag"
+          matchExpressions:
+            "flag_1": {op: Exists}
+
+    # Negative test not supposed to create a label
+    - name: "e2e-flag-test-neg-1"
+      labels:
+        e2e-flag-test-neg-1: "true"
+      matchFeatures:
+        - feature: "fake.flag"
+          matchExpressions:
+            "flag_1": {op: DoesNotExist}
+
+    #
+    # Simple test rules for attribute features
+    #
+    - name: "e2e-attribute-test-1"
+      labels:
+        e2e-attribute-test-1: "true"
+      vars:
+        e2e-attribute-test-1.not: "false"
+      matchFeatures:
+        - feature: "fake.attribute"
+          matchExpressions:
+            "attr_1": {op: IsTrue}
+            "attr_2": {op: IsFalse}
+
+    # Negative test not supposed to create a label
+    - name: "e2e-attribute-test-neg-1"
+      labels:
+        e2e-attribute-test-neg-1: "true"
+      matchFeatures:
+        - feature: "fake.attribute"
+          matchExpressions:
+            "attr_1": {op: IsTrue}
+            "attr_2": {op: IsTrue}
+
+    #
+    # Simple test rules for instnace features
+    #
+    - name: "e2e-instance-test-1"
+      labels:
+        e2e-instance-test-1: "true"
+      vars:
+        e2e-instance-test-1.not: "false"
+        e2e-instance-test-1.123: "123"
+      matchFeatures:
+        - feature: "fake.instance"
+          matchExpressions:
+            "attr_1": {op: In, value: ["true"]}
+            "attr_3": {op: Gt, value: ["10"]}
+
+    # Negative test not supposed to create a label
+    - name: "e2e-instance-test-neg-1"
+      labels:
+        e2e-instance-test-neg-1: "true"
+      matchFeatures:
+        - feature: "fake.instance"
+          matchExpressions:
+            "attr_1": {op: In, value: ["true"]}
+            "attr_3": {op: Lt, value: ["10"]}

test/e2e/data/nodefeaturerule-2.yaml

@@ -0,0 +1,41 @@
+apiVersion: nfd.k8s-sigs.io/v1alpha1
+kind: NodeFeatureRule
+metadata:
+  name: e2e-test-2
+spec:
+  rules:
+    #
+    # More complex rule testing backreferencing and matchAny field
+    #
+    - name: "e2e-matchany-test-1"
+      labels:
+        e2e-matchany-test-1: "true"
+      vars:
+        e2e-instance-test-1.not: "false"
+      matchFeatures:
+        - feature: "rule.matched"
+          matchExpressions:
+            "e2e-attribute-test-1": {op: InRegexp, value: ["^tru"]}
+            "e2e-instance-test-1.123": {op: In, value: ["1", "12", "123"]}
+      matchAny:
+        - matchFeatures:
+            - feature: "fake.instance"
+              matchExpressions:
+                "attr_1": {op: In, value: ["nomatch"]}
+        - matchFeatures:
+            - feature: "fake.instance"
+              matchExpressions:
+                "attr_3": {op: In, value: ["100"]}
+
+    #
+    # Simple test for templating
+    #
+    - name: "e2e-template-test-1"
+      labelsTemplate: |
+        {{ range .fake.instance }}e2e-template-test-1-{{ .name }}=found
+        {{ end }}
+      matchFeatures:
+        - feature: "fake.instance"
+          matchExpressions:
+            "attr_1": {op: In, value: ["true"]}
+

test/e2e/node_feature_discovery.go

@@ -24,10 +24,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	v1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	clientset "k8s.io/client-go/kubernetes"
@@ -35,6 +38,7 @@ import (
 	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
 	e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+	nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned"
 
 	nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1"
 	"sigs.k8s.io/node-feature-discovery/source/custom"
@@ -419,6 +423,103 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
 			})
 		})
 
-	})
+		//
+		// Test NodeFeatureRule
+		//
+		Context("and nfd-worker and NodeFeatureRules objects deployed", func() {
+			var extClient *extclient.Clientset
+			var nfdClient *nfdclient.Clientset
+			var crd *apiextensionsv1.CustomResourceDefinition
 
+			BeforeEach(func() {
+				// Create clients for apiextensions and our CRD api
+				extClient = extclient.NewForConfigOrDie(f.ClientConfig())
+				nfdClient = nfdclient.NewForConfigOrDie(f.ClientConfig())
+
+				// Create CRDs
+				By("Creating NodeFeatureRule CRD")
+				var err error
+				crd, err = testutils.CreateNodeFeatureRulesCRD(extClient)
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			AfterEach(func() {
+				err := extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{})
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It("custom labels from the NodeFeatureRule rules should be created", func() {
+				By("Creating nfd-worker daemonset")
+				workerArgs := []string{"-feature-sources=fake", "-label-sources=", "-sleep-interval=1s"}
+				workerDS := testutils.NFDWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), workerArgs)
+				workerDS, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{})
+				Expect(err).NotTo(HaveOccurred())
+
+				By("Waiting for daemonset pods to be ready")
+				Expect(testutils.WaitForPodsReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred())
+
+				expected := map[string]string{
+					"feature.node.kubernetes.io/e2e-flag-test-1":      "true",
+					"feature.node.kubernetes.io/e2e-attribute-test-1": "true",
+					"feature.node.kubernetes.io/e2e-instance-test-1":  "true"}
+
+				By("Creating NodeFeatureRules #1")
+				Expect(testutils.CreateNodeFeatureRuleFromFile(nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred())
+
+				By("Verifying node labels from NodeFeatureRules #1")
+				Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred())
+
+				By("Creating NodeFeatureRules #2")
+				Expect(testutils.CreateNodeFeatureRuleFromFile(nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred())
+
+				// Add features from NodeFeatureRule #2
+				expected["feature.node.kubernetes.io/e2e-matchany-test-1"] = "true"
+				expected["feature.node.kubernetes.io/e2e-template-test-1-instance_1"] = "found"
+				expected["feature.node.kubernetes.io/e2e-template-test-1-instance_2"] = "found"
+
+				By("Verifying node labels from NodeFeatureRules #1 and #2")
+				Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred())
+			})
+		})
+	})
 })
+
+// waitForNfdNodeLabels waits for node to be labeled as expected.
+func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) error {
+	poll := func() error {
+		nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+		if err != nil {
+			return err
+		}
+		for _, node := range nodeList.Items {
+			labels := nfdLabels(node.Labels)
+			if !cmp.Equal(expected, labels) {
+				return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(expected, labels))
+			}
+		}
+		return nil
+	}
+
+	// Simple and stupid re-try loop
+	var err error
+	for retry := 0; retry < 3; retry++ {
+		if err = poll(); err == nil {
+			return nil
+		}
+		time.Sleep(2 * time.Second)
+	}
+	return err
+}
+
+// nfdLabels gets labels that are in the nfd label namespace.
+func nfdLabels(labels map[string]string) map[string]string {
+	ret := map[string]string{}
+
+	for key, val := range labels {
+		if strings.HasPrefix(key, nfdv1alpha1.FeatureLabelNs) {
+			ret[key] = val
+		}
+	}
+	return ret
+
+}

test/e2e/utils/crd.go

@@ -0,0 +1,110 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apiruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/scheme"
+	nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1"
+	nfdclientset "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned"
+	nfdscheme "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned/scheme"
+)
+
+var packagePath string
+
+// CreateNodeFeatureRulesCRD creates the NodeFeatureRule CRD in the API server.
+func CreateNodeFeatureRulesCRD(cli extclient.Interface) (*apiextensionsv1.CustomResourceDefinition, error) {
+	crd, err := crdFromFile(filepath.Join(packagePath, "..", "..", "..", "deployment", "base", "nfd-crds", "nodefeaturerule-crd.yaml"))
+	if err != nil {
+		return nil, err
+	}
+
+	// Delete existing CRD (if any) with this we also get rid of stale objects
+	err = cli.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{})
+	if err != nil && !errors.IsNotFound(err) {
+		return nil, fmt.Errorf("failed to delete NodeFeatureRule CRD: %w", err)
+	}
+
+	return cli.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{})
+}
+
+// CreateNodeFeatureRuleFromFile creates a NodeFeatureRule object from a given file located under test data directory.
+func CreateNodeFeatureRuleFromFile(cli nfdclientset.Interface, filename string) error {
+	obj, err := nodeFeatureRuleFromFile(filepath.Join(packagePath, "..", "data", filename))
+	if err != nil {
+		return err
+	}
+	_, err = cli.NfdV1alpha1().NodeFeatureRules().Create(context.TODO(), obj, metav1.CreateOptions{})
+	return err
+}
+
+func apiObjFromFile(path string, decoder apiruntime.Decoder) (apiruntime.Object, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	obj, _, err := decoder.Decode(data, nil, nil)
+	return obj, err
+}
+
+// crdFromFile creates a CustomResourceDefinition API object from a file.
+func crdFromFile(path string) (*apiextensionsv1.CustomResourceDefinition, error) {
+	obj, err := apiObjFromFile(path, scheme.Codecs.UniversalDeserializer())
+	if err != nil {
+		return nil, err
+	}
+
+	crd, ok := obj.(*apiextensionsv1.CustomResourceDefinition)
+	if !ok {
+		return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path)
+	}
+
+	return crd, nil
+}
+
+func nodeFeatureRuleFromFile(path string) (*nfdv1alpha1.NodeFeatureRule, error) {
+	obj, err := apiObjFromFile(path, nfdscheme.Codecs.UniversalDeserializer())
+	if err != nil {
+		return nil, err
+	}
+
+	crd, ok := obj.(*nfdv1alpha1.NodeFeatureRule)
+	if !ok {
+		return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path)
+	}
+
+	return crd, nil
+}
+
+func init() {
+	_, thisFile, _, _ := runtime.Caller(0)
+	packagePath = filepath.Dir(thisFile)
+
+	// Register k8s scheme to be able to create CRDs
+	_ = apiextensionsv1.AddToScheme(scheme.Scheme)
+}

test/e2e/utils/rbac.go

@@ -128,6 +128,11 @@ func createClusterRoleMaster(cs clientset.Interface) (*rbacv1.ClusterRole, error
 				Resources: []string{"nodes"},
 				Verbs:     []string{"get", "patch", "update"},
 			},
+			{
+				APIGroups: []string{"nfd.k8s-sigs.io"},
+				Resources: []string{"nodefeaturerules"},
+				Verbs:     []string{"get", "list", "watch"},
+			},
 			{
 				APIGroups: []string{"topology.node.k8s.io"},
 				Resources: []string{"noderesourcetopologies"},