2019-04-25 14:41:26 +00:00
|
|
|
/*
|
|
|
|
Copyright 2018 The Kubernetes Authors.
|
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package e2e
|
|
|
|
|
|
|
|
import (
|
2020-11-17 14:40:33 +00:00
|
|
|
"context"
|
2023-04-25 01:03:02 +00:00
|
|
|
"encoding/json"
|
2019-04-25 20:19:57 +00:00
|
|
|
"fmt"
|
2021-01-20 23:48:40 +00:00
|
|
|
"path/filepath"
|
2020-02-21 14:16:37 +00:00
|
|
|
"strings"
|
2019-04-25 20:19:57 +00:00
|
|
|
"time"
|
|
|
|
|
2022-09-07 16:25:57 +00:00
|
|
|
. "github.com/onsi/ginkgo/v2"
|
2021-01-20 23:48:40 +00:00
|
|
|
. "github.com/onsi/gomega"
|
2020-11-17 14:40:33 +00:00
|
|
|
|
2022-10-14 12:28:52 +00:00
|
|
|
corev1 "k8s.io/api/core/v1"
|
2022-07-07 07:40:19 +00:00
|
|
|
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
|
|
|
extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
2023-03-10 10:39:41 +00:00
|
|
|
resourcev1 "k8s.io/apimachinery/pkg/api/resource"
|
2019-04-25 20:19:57 +00:00
|
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
2023-04-25 01:03:02 +00:00
|
|
|
"k8s.io/apimachinery/pkg/types"
|
2019-04-25 20:19:57 +00:00
|
|
|
clientset "k8s.io/client-go/kubernetes"
|
2022-12-16 20:10:36 +00:00
|
|
|
taintutils "k8s.io/kubernetes/pkg/util/taints"
|
2019-04-25 14:41:26 +00:00
|
|
|
"k8s.io/kubernetes/test/e2e/framework"
|
2022-12-12 14:29:46 +00:00
|
|
|
e2elog "k8s.io/kubernetes/test/e2e/framework"
|
2020-11-17 14:40:33 +00:00
|
|
|
e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
|
2020-02-05 15:21:36 +00:00
|
|
|
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
2022-12-15 14:40:20 +00:00
|
|
|
admissionapi "k8s.io/pod-security-admission/api"
|
2022-11-24 10:59:38 +00:00
|
|
|
|
2023-04-25 01:03:02 +00:00
|
|
|
"sigs.k8s.io/node-feature-discovery/pkg/apihelper"
|
2022-08-16 18:39:11 +00:00
|
|
|
nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1"
|
2022-11-17 14:57:20 +00:00
|
|
|
nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned"
|
2021-01-20 23:48:40 +00:00
|
|
|
"sigs.k8s.io/node-feature-discovery/source/custom"
|
2020-11-23 16:37:42 +00:00
|
|
|
testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils"
|
2022-11-24 11:23:34 +00:00
|
|
|
testds "sigs.k8s.io/node-feature-discovery/test/e2e/utils/daemonset"
|
2022-11-24 10:59:38 +00:00
|
|
|
testpod "sigs.k8s.io/node-feature-discovery/test/e2e/utils/pod"
|
2019-04-25 20:19:57 +00:00
|
|
|
)
|
|
|
|
|
2020-02-21 14:16:37 +00:00
|
|
|
// cleanupNode deletes all NFD-related metadata from the Node object, i.e.
|
|
|
|
// labels and annotations
|
2023-04-18 11:23:03 +00:00
|
|
|
func cleanupNode(ctx context.Context, cs clientset.Interface) {
|
2023-04-05 11:46:39 +00:00
|
|
|
// Per-node cleanup function
|
|
|
|
cleanup := func(nodeName string) error {
|
2023-04-18 11:23:03 +00:00
|
|
|
node, err := cs.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
|
2023-04-05 11:46:39 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
2020-02-21 14:16:37 +00:00
|
|
|
|
2023-04-05 11:46:39 +00:00
|
|
|
update := false
|
|
|
|
updateStatus := false
|
|
|
|
// Gather info about all NFD-managed node assets outside the default prefix
|
|
|
|
nfdLabels := map[string]struct{}{}
|
|
|
|
for _, name := range strings.Split(node.Annotations[nfdv1alpha1.FeatureLabelsAnnotation], ",") {
|
|
|
|
if strings.Contains(name, "/") {
|
|
|
|
nfdLabels[name] = struct{}{}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
nfdERs := map[string]struct{}{}
|
|
|
|
for _, name := range strings.Split(node.Annotations[nfdv1alpha1.ExtendedResourceAnnotation], ",") {
|
|
|
|
if strings.Contains(name, "/") {
|
|
|
|
nfdERs[name] = struct{}{}
|
2020-02-21 14:16:37 +00:00
|
|
|
}
|
2023-04-05 11:46:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Remove labels
|
|
|
|
for key := range node.Labels {
|
|
|
|
_, ok := nfdLabels[key]
|
|
|
|
if ok || strings.HasPrefix(key, nfdv1alpha1.FeatureLabelNs) {
|
|
|
|
delete(node.Labels, key)
|
|
|
|
update = true
|
|
|
|
}
|
|
|
|
}
|
2019-04-25 20:19:57 +00:00
|
|
|
|
2023-04-05 11:46:39 +00:00
|
|
|
// Remove annotations
|
|
|
|
for key := range node.Annotations {
|
|
|
|
if strings.HasPrefix(key, nfdv1alpha1.AnnotationNs) {
|
|
|
|
delete(node.Annotations, key)
|
|
|
|
update = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Remove taints
|
|
|
|
for _, taint := range node.Spec.Taints {
|
2023-04-05 12:27:50 +00:00
|
|
|
if strings.HasPrefix(taint.Key, nfdv1alpha1.TaintNs) {
|
2023-04-05 11:46:39 +00:00
|
|
|
newTaints, removed := taintutils.DeleteTaint(node.Spec.Taints, &taint)
|
|
|
|
if removed {
|
|
|
|
node.Spec.Taints = newTaints
|
2020-02-21 14:16:37 +00:00
|
|
|
update = true
|
|
|
|
}
|
2019-04-25 20:19:57 +00:00
|
|
|
}
|
2023-04-05 11:46:39 +00:00
|
|
|
}
|
2019-04-25 20:19:57 +00:00
|
|
|
|
2023-04-05 11:46:39 +00:00
|
|
|
// Remove extended resources
|
|
|
|
for key := range node.Status.Capacity {
|
|
|
|
// We check for FeatureLabelNs as -resource-labels can create ERs there
|
|
|
|
_, ok := nfdERs[string(key)]
|
|
|
|
if ok || strings.HasPrefix(string(key), nfdv1alpha1.FeatureLabelNs) {
|
|
|
|
delete(node.Status.Capacity, key)
|
|
|
|
delete(node.Status.Allocatable, key)
|
|
|
|
updateStatus = true
|
2022-11-30 23:21:21 +00:00
|
|
|
}
|
2023-04-05 11:46:39 +00:00
|
|
|
}
|
2022-11-30 23:21:21 +00:00
|
|
|
|
2023-04-05 11:46:39 +00:00
|
|
|
if updateStatus {
|
|
|
|
By("Deleting NFD extended resources from node " + nodeName)
|
2023-04-18 11:23:03 +00:00
|
|
|
if _, err := cs.CoreV1().Nodes().UpdateStatus(ctx, node, metav1.UpdateOptions{}); err != nil {
|
2023-04-05 11:46:39 +00:00
|
|
|
return err
|
2020-02-21 14:16:37 +00:00
|
|
|
}
|
2023-04-05 11:46:39 +00:00
|
|
|
}
|
2020-02-21 14:16:37 +00:00
|
|
|
|
2023-04-05 11:46:39 +00:00
|
|
|
if update {
|
2022-11-30 23:21:21 +00:00
|
|
|
By("Deleting NFD labels, annotations and taints from node " + node.Name)
|
2023-04-18 11:23:03 +00:00
|
|
|
if _, err := cs.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}); err != nil {
|
2023-04-05 11:46:39 +00:00
|
|
|
return err
|
2020-02-21 14:16:37 +00:00
|
|
|
}
|
2023-04-05 11:46:39 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Cleanup all nodes
|
2023-04-18 11:23:03 +00:00
|
|
|
nodeList, err := cs.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
|
2023-04-05 11:46:39 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
2020-02-21 14:16:37 +00:00
|
|
|
|
2023-04-05 11:46:39 +00:00
|
|
|
for _, n := range nodeList.Items {
|
|
|
|
var err error
|
|
|
|
for retry := 0; retry < 5; retry++ {
|
|
|
|
if err = cleanup(n.Name); err == nil {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
time.Sleep(100 * time.Millisecond)
|
2020-02-21 14:16:37 +00:00
|
|
|
}
|
2021-01-20 23:48:40 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
2020-02-21 14:16:37 +00:00
|
|
|
}
|
2022-12-14 16:17:20 +00:00
|
|
|
}
|
|
|
|
|
2023-04-18 11:23:03 +00:00
|
|
|
func cleanupCRs(ctx context.Context, cli *nfdclient.Clientset, namespace string) {
|
2022-12-14 16:17:20 +00:00
|
|
|
// Drop NodeFeatureRule objects
|
2023-04-18 11:23:03 +00:00
|
|
|
nfrs, err := cli.NfdV1alpha1().NodeFeatureRules().List(ctx, metav1.ListOptions{})
|
2022-12-14 16:17:20 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
By("Deleting NodeFeatureRule objects from the cluster")
|
|
|
|
for _, nfr := range nfrs.Items {
|
2023-04-18 11:23:03 +00:00
|
|
|
err = cli.NfdV1alpha1().NodeFeatureRules().Delete(ctx, nfr.Name, metav1.DeleteOptions{})
|
2022-12-14 16:17:20 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
}
|
2023-03-07 09:02:43 +00:00
|
|
|
|
2023-04-18 11:23:03 +00:00
|
|
|
nfs, err := cli.NfdV1alpha1().NodeFeatures(namespace).List(ctx, metav1.ListOptions{})
|
2023-03-07 09:02:43 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
By("Deleting NodeFeature objects from namespace " + namespace)
|
|
|
|
for _, nf := range nfs.Items {
|
2023-04-18 11:23:03 +00:00
|
|
|
err = cli.NfdV1alpha1().NodeFeatures(namespace).Delete(ctx, nf.Name, metav1.DeleteOptions{})
|
2023-03-07 09:02:43 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
}
|
2020-02-21 14:16:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Actual test suite
|
2023-02-21 11:37:25 +00:00
|
|
|
var _ = SIGDescribe("NFD master and worker", func() {
|
2020-02-21 14:16:37 +00:00
|
|
|
f := framework.NewDefaultFramework("node-feature-discovery")
|
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
nfdTestSuite := func(useNodeFeatureApi bool) {
|
2022-12-14 09:46:29 +00:00
|
|
|
createPodSpecOpts := func(opts ...testpod.SpecOption) []testpod.SpecOption {
|
|
|
|
if useNodeFeatureApi {
|
|
|
|
return append(opts, testpod.SpecWithContainerExtraArgs("-enable-nodefeature-api"))
|
|
|
|
}
|
|
|
|
return opts
|
|
|
|
}
|
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
Context("when deploying a single nfd-master pod", Ordered, func() {
|
|
|
|
var (
|
2023-04-05 17:42:20 +00:00
|
|
|
crds []*apiextensionsv1.CustomResourceDefinition
|
|
|
|
extClient *extclient.Clientset
|
|
|
|
nfdClient *nfdclient.Clientset
|
|
|
|
extraMasterPodSpecOpts []testpod.SpecOption
|
2022-12-14 14:06:04 +00:00
|
|
|
)
|
|
|
|
|
2023-04-18 11:23:03 +00:00
|
|
|
checkNodeFeatureObject := func(ctx context.Context, name string) {
|
|
|
|
_, err := nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Get(ctx, name, metav1.GetOptions{})
|
2022-12-14 09:46:29 +00:00
|
|
|
if useNodeFeatureApi {
|
|
|
|
By(fmt.Sprintf("Check that NodeFeature object for the node %q was created", name))
|
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
} else {
|
|
|
|
By(fmt.Sprintf("Check that NodeFeature object for the node %q hasn't been created", name))
|
|
|
|
Expect(err).To(HaveOccurred())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-04-18 11:23:03 +00:00
|
|
|
BeforeAll(func(ctx context.Context) {
|
2022-12-14 14:06:04 +00:00
|
|
|
// Create clients for apiextensions and our CRD api
|
|
|
|
extClient = extclient.NewForConfigOrDie(f.ClientConfig())
|
|
|
|
nfdClient = nfdclient.NewForConfigOrDie(f.ClientConfig())
|
|
|
|
|
|
|
|
By("Creating NFD CRDs")
|
|
|
|
var err error
|
2023-04-18 11:23:03 +00:00
|
|
|
crds, err = testutils.CreateNfdCRDs(ctx, extClient)
|
2022-12-15 14:40:20 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
2022-12-14 14:06:04 +00:00
|
|
|
})
|
2022-06-29 11:06:59 +00:00
|
|
|
|
2023-04-18 11:23:03 +00:00
|
|
|
AfterAll(func(ctx context.Context) {
|
2022-12-14 14:06:04 +00:00
|
|
|
for _, crd := range crds {
|
2023-04-18 11:23:03 +00:00
|
|
|
err := extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, crd.Name, metav1.DeleteOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
}
|
|
|
|
})
|
2019-04-25 20:19:57 +00:00
|
|
|
|
2023-04-18 11:23:03 +00:00
|
|
|
JustBeforeEach(func(ctx context.Context) {
|
2022-12-14 14:06:04 +00:00
|
|
|
// Drop the pod security admission label as nfd-worker needs host mounts
|
|
|
|
if _, ok := f.Namespace.Labels[admissionapi.EnforceLevelLabel]; ok {
|
|
|
|
e2elog.Logf("Deleting %s label from the test namespace", admissionapi.EnforceLevelLabel)
|
|
|
|
delete(f.Namespace.Labels, admissionapi.EnforceLevelLabel)
|
2023-04-18 11:23:03 +00:00
|
|
|
_, err := f.ClientSet.CoreV1().Namespaces().Update(ctx, f.Namespace, metav1.UpdateOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
}
|
2019-04-25 20:19:57 +00:00
|
|
|
|
2023-04-18 11:23:03 +00:00
|
|
|
err := testutils.ConfigureRBAC(ctx, f.ClientSet, f.Namespace.Name)
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
2019-04-25 20:19:57 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
// Remove pre-existing stale annotations and labels etc and CRDs
|
2023-04-18 11:23:03 +00:00
|
|
|
cleanupCRs(ctx, nfdClient, f.Namespace.Name)
|
|
|
|
cleanupNode(ctx, f.ClientSet)
|
2020-02-21 14:16:37 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
// Launch nfd-master
|
|
|
|
By("Creating nfd master pod and nfd-master service")
|
2023-04-05 17:42:20 +00:00
|
|
|
podSpecOpts := createPodSpecOpts(
|
|
|
|
append(extraMasterPodSpecOpts,
|
2023-03-05 21:56:46 +00:00
|
|
|
testpod.SpecWithContainerImage(dockerImage()),
|
2023-04-05 17:42:20 +00:00
|
|
|
)...)
|
2023-03-05 21:56:46 +00:00
|
|
|
|
2023-04-18 12:47:20 +00:00
|
|
|
masterPod := e2epod.NewPodClient(f).CreateSync(ctx, testpod.NFDMaster(podSpecOpts...))
|
2020-02-21 14:16:37 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
// Create nfd-master service
|
2023-04-18 11:23:03 +00:00
|
|
|
nfdSvc, err := testutils.CreateService(ctx, f.ClientSet, f.Namespace.Name)
|
2021-01-20 23:48:40 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
2020-02-21 14:16:37 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
By("Waiting for the nfd-master pod to be running")
|
2023-04-18 12:47:20 +00:00
|
|
|
Expect(e2epod.WaitTimeoutForPodRunningInNamespace(ctx, f.ClientSet, masterPod.Name, masterPod.Namespace, time.Minute)).NotTo(HaveOccurred())
|
2020-02-21 14:16:37 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
By("Verifying the node where nfd-master is running")
|
|
|
|
// Get updated masterPod object (we want to know where it was scheduled)
|
2023-04-18 11:23:03 +00:00
|
|
|
masterPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, masterPod.Name, metav1.GetOptions{})
|
2021-01-20 23:48:40 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
2022-12-14 14:06:04 +00:00
|
|
|
// Node running nfd-master should have master version annotation
|
2023-04-18 11:23:03 +00:00
|
|
|
masterPodNode, err := f.ClientSet.CoreV1().Nodes().Get(ctx, masterPod.Spec.NodeName, metav1.GetOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
Expect(masterPodNode.Annotations).To(HaveKey(nfdv1alpha1.AnnotationNs + "/master.version"))
|
2020-11-23 16:37:42 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
By("Waiting for the nfd-master service to be up")
|
2023-04-18 12:47:20 +00:00
|
|
|
Expect(e2enetwork.WaitForService(ctx, f.ClientSet, f.Namespace.Name, nfdSvc.Name, true, time.Second, 10*time.Second)).NotTo(HaveOccurred())
|
2022-12-14 14:06:04 +00:00
|
|
|
})
|
2020-02-21 14:16:37 +00:00
|
|
|
|
2023-04-18 11:23:03 +00:00
|
|
|
AfterEach(func(ctx context.Context) {
|
|
|
|
Expect(testutils.DeconfigureRBAC(ctx, f.ClientSet, f.Namespace.Name)).NotTo(HaveOccurred())
|
2020-02-21 14:16:37 +00:00
|
|
|
|
2023-04-18 11:23:03 +00:00
|
|
|
cleanupNode(ctx, f.ClientSet)
|
|
|
|
cleanupCRs(ctx, nfdClient, f.Namespace.Name)
|
2023-04-05 17:42:20 +00:00
|
|
|
extraMasterPodSpecOpts = nil
|
2022-12-14 14:06:04 +00:00
|
|
|
})
|
2020-02-21 14:16:37 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
//
|
|
|
|
// Simple test with only the fake source enabled
|
|
|
|
//
|
|
|
|
Context("and a single worker pod with fake source enabled", func() {
|
2023-04-18 11:23:03 +00:00
|
|
|
It("it should decorate the node with the fake feature labels", func(ctx context.Context) {
|
2023-04-20 15:09:50 +00:00
|
|
|
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
|
|
|
|
Expect(err).NotTo(HaveOccurred())
|
2020-02-21 14:16:37 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
// Launch nfd-worker
|
|
|
|
By("Creating a nfd worker pod")
|
2022-12-14 09:46:29 +00:00
|
|
|
podSpecOpts := createPodSpecOpts(
|
2022-12-14 14:06:04 +00:00
|
|
|
testpod.SpecWithRestartPolicy(corev1.RestartPolicyNever),
|
2023-01-11 10:14:43 +00:00
|
|
|
testpod.SpecWithContainerImage(dockerImage()),
|
2022-12-14 14:06:04 +00:00
|
|
|
testpod.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"),
|
2022-12-14 09:46:29 +00:00
|
|
|
)
|
2022-12-14 14:06:04 +00:00
|
|
|
workerPod := testpod.NFDWorker(podSpecOpts...)
|
2023-04-20 15:09:50 +00:00
|
|
|
workerPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, workerPod, metav1.CreateOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
By("Waiting for the nfd-worker pod to succeed")
|
2023-04-18 12:47:20 +00:00
|
|
|
Expect(e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, workerPod.Name, f.Namespace.Name)).NotTo(HaveOccurred())
|
2023-04-18 11:23:03 +00:00
|
|
|
workerPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, workerPod.Name, metav1.GetOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
By(fmt.Sprintf("Making sure '%s' was decorated with the fake feature labels", workerPod.Spec.NodeName))
|
2023-04-20 15:09:50 +00:00
|
|
|
expectedLabels := map[string]k8sLabels{
|
|
|
|
workerPod.Spec.NodeName: {
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature1": "true",
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2": "true",
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "true",
|
|
|
|
},
|
|
|
|
"*": {},
|
2020-02-21 14:16:37 +00:00
|
|
|
}
|
2023-04-20 15:09:50 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2020-02-21 14:16:37 +00:00
|
|
|
|
2023-04-20 15:09:50 +00:00
|
|
|
checkNodeFeatureObject(ctx, workerPod.Spec.NodeName)
|
2022-12-14 09:46:29 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
By("Deleting the node-feature-discovery worker pod")
|
2023-04-18 11:23:03 +00:00
|
|
|
err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, workerPod.Name, metav1.DeleteOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
})
|
|
|
|
})
|
|
|
|
|
|
|
|
//
|
|
|
|
// More comprehensive test when --e2e-node-config is enabled
|
|
|
|
//
|
|
|
|
Context("and nfd-workers as a daemonset with default sources enabled", func() {
|
2023-04-18 11:23:03 +00:00
|
|
|
It("the node labels and annotations listed in the e2e config should be present", func(ctx context.Context) {
|
2022-12-14 14:06:04 +00:00
|
|
|
cfg, err := testutils.GetConfig()
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
|
|
|
|
if cfg == nil {
|
|
|
|
Skip("no e2e-config was specified")
|
2020-02-21 14:16:37 +00:00
|
|
|
}
|
2022-12-14 14:06:04 +00:00
|
|
|
if cfg.DefaultFeatures == nil {
|
|
|
|
Skip("no 'defaultFeatures' specified in e2e-config")
|
2020-02-21 14:16:37 +00:00
|
|
|
}
|
2022-12-14 14:06:04 +00:00
|
|
|
fConf := cfg.DefaultFeatures
|
|
|
|
|
|
|
|
By("Creating nfd-worker daemonset")
|
2022-12-14 09:46:29 +00:00
|
|
|
podSpecOpts := createPodSpecOpts(
|
2023-01-11 10:14:43 +00:00
|
|
|
testpod.SpecWithContainerImage(dockerImage()),
|
2022-12-14 09:46:29 +00:00
|
|
|
)
|
2022-12-14 14:06:04 +00:00
|
|
|
workerDS := testds.NFDWorker(podSpecOpts...)
|
2023-04-18 11:23:03 +00:00
|
|
|
workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(ctx, workerDS, metav1.CreateOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
2023-03-07 09:11:52 +00:00
|
|
|
By("Waiting for worker daemonset pods to be ready")
|
2023-04-18 11:23:03 +00:00
|
|
|
Expect(testpod.WaitForReady(ctx, f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 2)).NotTo(HaveOccurred())
|
2022-12-14 14:06:04 +00:00
|
|
|
|
|
|
|
By("Getting node objects")
|
2023-04-18 11:23:03 +00:00
|
|
|
nodeList, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
Expect(len(nodeList.Items)).ToNot(BeZero())
|
|
|
|
|
|
|
|
for _, node := range nodeList.Items {
|
|
|
|
nodeConf := testutils.FindNodeConfig(cfg, node.Name)
|
|
|
|
if nodeConf == nil {
|
|
|
|
e2elog.Logf("node %q has no matching rule in e2e-config, skipping...", node.Name)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check labels
|
|
|
|
e2elog.Logf("verifying labels of node %q...", node.Name)
|
|
|
|
for k, v := range nodeConf.ExpectedLabelValues {
|
|
|
|
Expect(node.Labels).To(HaveKeyWithValue(k, v))
|
|
|
|
}
|
|
|
|
for k := range nodeConf.ExpectedLabelKeys {
|
|
|
|
Expect(node.Labels).To(HaveKey(k))
|
|
|
|
}
|
|
|
|
for k := range node.Labels {
|
|
|
|
if strings.HasPrefix(k, nfdv1alpha1.FeatureLabelNs) {
|
|
|
|
if _, ok := nodeConf.ExpectedLabelValues[k]; ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if _, ok := nodeConf.ExpectedLabelKeys[k]; ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
// Ignore if the label key was not whitelisted
|
|
|
|
Expect(fConf.LabelWhitelist).NotTo(HaveKey(k))
|
2020-02-21 14:16:37 +00:00
|
|
|
}
|
2022-12-14 14:06:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Check annotations
|
|
|
|
e2elog.Logf("verifying annotations of node %q...", node.Name)
|
|
|
|
for k, v := range nodeConf.ExpectedAnnotationValues {
|
|
|
|
Expect(node.Annotations).To(HaveKeyWithValue(k, v))
|
|
|
|
}
|
|
|
|
for k := range nodeConf.ExpectedAnnotationKeys {
|
|
|
|
Expect(node.Annotations).To(HaveKey(k))
|
|
|
|
}
|
|
|
|
for k := range node.Annotations {
|
|
|
|
if strings.HasPrefix(k, nfdv1alpha1.AnnotationNs) {
|
|
|
|
if _, ok := nodeConf.ExpectedAnnotationValues[k]; ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if _, ok := nodeConf.ExpectedAnnotationKeys[k]; ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
// Ignore if the annotation was not whitelisted
|
|
|
|
Expect(fConf.AnnotationWhitelist).NotTo(HaveKey(k))
|
2020-02-21 14:16:37 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-12-14 09:46:29 +00:00
|
|
|
// Check existence of NodeFeature object
|
2023-04-18 11:23:03 +00:00
|
|
|
checkNodeFeatureObject(ctx, node.Name)
|
2022-12-14 09:46:29 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
}
|
2020-02-21 14:16:37 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
By("Deleting nfd-worker daemonset")
|
2023-04-18 11:23:03 +00:00
|
|
|
err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(ctx, workerDS.Name, metav1.DeleteOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
})
|
2020-02-21 14:16:37 +00:00
|
|
|
})
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
//
|
|
|
|
// Test custom nodename source configured in 2 additional ConfigMaps
|
|
|
|
//
|
|
|
|
Context("and nfd-workers as a daemonset with 2 additional configmaps for the custom source configured", func() {
|
2023-04-18 11:23:03 +00:00
|
|
|
It("the nodename matching features listed in the configmaps should be present", func(ctx context.Context) {
|
2022-12-14 14:06:04 +00:00
|
|
|
By("Getting a worker node")
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
// We need a valid nodename for the configmap
|
2023-04-18 11:23:03 +00:00
|
|
|
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
targetNodeName := nodes[0].Name
|
|
|
|
Expect(targetNodeName).ToNot(BeEmpty(), "No worker node found")
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
// create a wildcard name as well for this node
|
|
|
|
targetNodeNameWildcard := fmt.Sprintf("%s.*%s", targetNodeName[:2], targetNodeName[4:])
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
By("Creating the configmaps")
|
|
|
|
targetLabelName := "nodename-test"
|
|
|
|
targetLabelValue := "true"
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
targetLabelNameWildcard := "nodename-test-wildcard"
|
|
|
|
targetLabelValueWildcard := "customValue"
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
// create 2 configmaps
|
2023-04-20 15:09:50 +00:00
|
|
|
data1 := fmt.Sprintf(`
|
|
|
|
- name: %s
|
2021-01-20 23:48:40 +00:00
|
|
|
matchOn:
|
|
|
|
# default value is true
|
|
|
|
- nodename:
|
2023-04-20 15:09:50 +00:00
|
|
|
- "^%s$"`, targetLabelName, targetNodeName)
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
cm1 := testutils.NewConfigMap("custom-config-extra-1", "custom.conf", data1)
|
2023-04-18 11:23:03 +00:00
|
|
|
cm1, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm1, metav1.CreateOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2023-04-20 15:09:50 +00:00
|
|
|
data2 := fmt.Sprintf(`
|
|
|
|
- name: %s
|
|
|
|
value: %s
|
2021-01-20 23:48:40 +00:00
|
|
|
matchOn:
|
|
|
|
- nodename:
|
2023-04-20 15:09:50 +00:00
|
|
|
- "^%s$"
|
|
|
|
- name: nodename-test-negative
|
2021-01-20 23:48:40 +00:00
|
|
|
matchOn:
|
|
|
|
- nodename:
|
2023-04-20 15:09:50 +00:00
|
|
|
- "thisNameShouldNeverMatch"`, targetLabelNameWildcard, targetLabelValueWildcard, targetNodeNameWildcard)
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
cm2 := testutils.NewConfigMap("custom-config-extra-2", "custom.conf", data2)
|
2023-04-18 11:23:03 +00:00
|
|
|
cm2, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm2, metav1.CreateOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
By("Creating nfd-worker daemonset with configmap mounted")
|
2022-12-14 09:46:29 +00:00
|
|
|
podSpecOpts := createPodSpecOpts(
|
2023-01-11 10:14:43 +00:00
|
|
|
testpod.SpecWithContainerImage(dockerImage()),
|
2023-04-20 15:09:50 +00:00
|
|
|
testpod.SpecWithContainerExtraArgs("-label-sources=custom"),
|
2022-12-14 14:06:04 +00:00
|
|
|
testpod.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")),
|
|
|
|
testpod.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")),
|
2022-12-14 09:46:29 +00:00
|
|
|
)
|
2022-12-14 14:06:04 +00:00
|
|
|
workerDS := testds.NFDWorker(podSpecOpts...)
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2023-04-18 11:23:03 +00:00
|
|
|
workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(ctx, workerDS, metav1.CreateOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2023-03-07 09:11:52 +00:00
|
|
|
By("Waiting for worker daemonset pods to be ready")
|
2023-04-18 11:23:03 +00:00
|
|
|
Expect(testpod.WaitForReady(ctx, f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 2)).NotTo(HaveOccurred())
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2023-04-20 15:09:50 +00:00
|
|
|
By("Verifying node labels")
|
|
|
|
expectedLabels := map[string]k8sLabels{
|
|
|
|
targetNodeName: {
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/custom-" + targetLabelName: targetLabelValue,
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/custom-" + targetLabelNameWildcard: targetLabelValueWildcard,
|
|
|
|
},
|
|
|
|
"*": {},
|
2021-01-20 23:48:40 +00:00
|
|
|
}
|
2023-04-20 15:09:50 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2021-01-20 23:48:40 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
By("Deleting nfd-worker daemonset")
|
2023-04-18 11:23:03 +00:00
|
|
|
err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(ctx, workerDS.Name, metav1.DeleteOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
})
|
2021-01-20 23:48:40 +00:00
|
|
|
})
|
|
|
|
|
2022-12-22 13:49:43 +00:00
|
|
|
//
|
|
|
|
// Test NodeFeature
|
|
|
|
//
|
|
|
|
Context("and NodeFeature objects deployed", func() {
|
2023-04-18 11:23:03 +00:00
|
|
|
BeforeEach(func(ctx context.Context) {
|
2023-04-05 17:42:20 +00:00
|
|
|
extraMasterPodSpecOpts = []testpod.SpecOption{
|
|
|
|
testpod.SpecWithContainerExtraArgs(
|
|
|
|
"-deny-label-ns=*.denied.ns,random.unwanted.ns,*.vendor.io",
|
|
|
|
"-extra-label-ns=custom.vendor.io",
|
|
|
|
),
|
|
|
|
}
|
|
|
|
})
|
2023-04-18 11:23:03 +00:00
|
|
|
It("labels from the NodeFeature objects should be created", func(ctx context.Context) {
|
2022-12-22 13:49:43 +00:00
|
|
|
if !useNodeFeatureApi {
|
|
|
|
Skip("NodeFeature API not enabled")
|
|
|
|
}
|
|
|
|
|
|
|
|
// We pick one node targeted for our NodeFeature objects
|
2023-04-18 11:23:03 +00:00
|
|
|
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
|
2022-12-22 13:49:43 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
targetNodeName := nodes[0].Name
|
|
|
|
Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
|
|
|
|
|
|
|
|
By("Creating NodeFeature object")
|
2023-04-18 11:23:03 +00:00
|
|
|
nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(ctx, nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName)
|
2022-12-22 13:49:43 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
By("Verifying node labels from NodeFeature object #1")
|
|
|
|
expectedLabels := map[string]k8sLabels{
|
|
|
|
targetNodeName: {
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-1": "obj-1",
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-2": "obj-1",
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "overridden",
|
|
|
|
},
|
2023-04-20 15:09:50 +00:00
|
|
|
"*": {},
|
2022-12-22 13:49:43 +00:00
|
|
|
}
|
2023-04-20 15:09:50 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2022-12-22 13:49:43 +00:00
|
|
|
By("Deleting NodeFeature object")
|
2023-04-18 11:23:03 +00:00
|
|
|
err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{})
|
2022-12-22 13:49:43 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
By("Verifying node labels from NodeFeature object were removed")
|
2023-04-20 15:09:50 +00:00
|
|
|
expectedLabels[targetNodeName] = k8sLabels{}
|
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2022-12-22 13:49:43 +00:00
|
|
|
|
|
|
|
By("Creating nfd-worker daemonset")
|
|
|
|
podSpecOpts := createPodSpecOpts(
|
2023-01-11 10:14:43 +00:00
|
|
|
testpod.SpecWithContainerImage(dockerImage()),
|
2022-12-22 13:49:43 +00:00
|
|
|
testpod.SpecWithContainerExtraArgs("-label-sources=fake"),
|
|
|
|
)
|
|
|
|
workerDS := testds.NFDWorker(podSpecOpts...)
|
2023-04-18 11:23:03 +00:00
|
|
|
workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(ctx, workerDS, metav1.CreateOptions{})
|
2022-12-22 13:49:43 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
By("Waiting for worker daemonset pods to be ready")
|
2023-04-18 11:23:03 +00:00
|
|
|
Expect(testpod.WaitForReady(ctx, f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 2)).NotTo(HaveOccurred())
|
2022-12-22 13:49:43 +00:00
|
|
|
|
|
|
|
By("Verifying node labels from nfd-worker")
|
|
|
|
expectedLabels = map[string]k8sLabels{
|
|
|
|
"*": {
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature1": "true",
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2": "true",
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "true",
|
|
|
|
},
|
|
|
|
}
|
2023-04-20 15:09:50 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2022-12-22 13:49:43 +00:00
|
|
|
|
|
|
|
By("Re-creating NodeFeature object")
|
2023-04-18 11:23:03 +00:00
|
|
|
_, err = testutils.CreateOrUpdateNodeFeaturesFromFile(ctx, nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName)
|
2022-12-22 13:49:43 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
By("Verifying node labels from NodeFeature object #1 are created")
|
|
|
|
expectedLabels[targetNodeName] = k8sLabels{
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-1": "obj-1",
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-2": "obj-1",
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature1": "true",
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2": "true",
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "overridden",
|
|
|
|
}
|
2023-04-20 15:09:50 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2022-12-22 13:49:43 +00:00
|
|
|
|
|
|
|
By("Creating extra namespace")
|
2023-04-18 12:47:20 +00:00
|
|
|
extraNs, err := f.CreateNamespace(ctx, "node-feature-discvery-extra-ns", nil)
|
2022-12-22 13:49:43 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
By("Create NodeFeature object in the extra namespace")
|
2023-04-18 11:23:03 +00:00
|
|
|
nodeFeatures, err = testutils.CreateOrUpdateNodeFeaturesFromFile(ctx, nfdClient, "nodefeature-2.yaml", extraNs.Name, targetNodeName)
|
2022-12-22 13:49:43 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
By("Verifying node labels from NodeFeature object #2 are created")
|
|
|
|
expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-1"] = "overridden-from-obj-2"
|
|
|
|
expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-3"] = "obj-2"
|
2023-04-20 15:09:50 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2023-03-07 09:02:43 +00:00
|
|
|
|
|
|
|
By("Deleting NodeFeature object from the extra namespace")
|
2023-04-18 11:23:03 +00:00
|
|
|
err = nfdClient.NfdV1alpha1().NodeFeatures(extraNs.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{})
|
2023-03-07 09:02:43 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
By("Verifying node labels from NodeFeature object were removed")
|
|
|
|
expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-1"] = "obj-1"
|
|
|
|
delete(expectedLabels[targetNodeName], nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-3")
|
2023-04-20 15:09:50 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2023-02-25 11:42:39 +00:00
|
|
|
})
|
|
|
|
|
2023-04-18 11:23:03 +00:00
|
|
|
It("denied labels should not be created by the NodeFeature object", func(ctx context.Context) {
|
2023-02-25 11:42:39 +00:00
|
|
|
if !useNodeFeatureApi {
|
|
|
|
Skip("NodeFeature API not enabled")
|
|
|
|
}
|
|
|
|
|
2023-04-18 11:23:03 +00:00
|
|
|
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
|
2023-02-25 11:42:39 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
targetNodeName := nodes[0].Name
|
|
|
|
Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
|
|
|
|
|
|
|
|
// Apply Node Feature object
|
|
|
|
By("Create NodeFeature object")
|
2023-04-18 11:23:03 +00:00
|
|
|
nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(ctx, nfdClient, "nodefeature-3.yaml", f.Namespace.Name, targetNodeName)
|
2023-02-25 11:42:39 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
// Verify that denied label was not added
|
|
|
|
By("Verifying that denied labels were not added")
|
|
|
|
expectedLabels := map[string]k8sLabels{
|
|
|
|
targetNodeName: {
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-4": "obj-4",
|
|
|
|
"custom.vendor.io/e2e-nodefeature-test-3": "vendor-ns",
|
|
|
|
},
|
|
|
|
}
|
2023-04-20 15:09:50 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2023-02-25 11:42:39 +00:00
|
|
|
|
|
|
|
By("Deleting NodeFeature object")
|
2023-04-18 11:23:03 +00:00
|
|
|
err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{})
|
2023-02-25 11:42:39 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
2023-04-20 15:09:50 +00:00
|
|
|
expectedLabels[targetNodeName] = k8sLabels{}
|
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2022-12-22 13:49:43 +00:00
|
|
|
})
|
|
|
|
})
|
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
// Test NodeFeatureRule
|
|
|
|
//
|
|
|
|
Context("and nfd-worker and NodeFeatureRules objects deployed", func() {
|
2023-04-05 17:42:20 +00:00
|
|
|
testTolerations := []corev1.Toleration{
|
|
|
|
{
|
2023-04-05 12:27:50 +00:00
|
|
|
Key: "feature.node.kubernetes.io/fake-special-node",
|
2023-04-05 17:42:20 +00:00
|
|
|
Value: "exists",
|
|
|
|
Effect: "NoExecute",
|
|
|
|
},
|
|
|
|
{
|
2023-04-05 12:27:50 +00:00
|
|
|
Key: "feature.node.kubernetes.io/fake-dedicated-node",
|
2023-04-05 17:42:20 +00:00
|
|
|
Value: "true",
|
|
|
|
Effect: "NoExecute",
|
|
|
|
},
|
|
|
|
{
|
2023-04-05 12:27:50 +00:00
|
|
|
Key: "feature.node.kubernetes.io/performance-optimized-node",
|
2023-04-05 17:42:20 +00:00
|
|
|
Value: "true",
|
|
|
|
Effect: "NoExecute",
|
|
|
|
},
|
|
|
|
{
|
2023-04-05 12:27:50 +00:00
|
|
|
Key: "feature.node.kubernetes.io/foo",
|
2023-04-05 17:42:20 +00:00
|
|
|
Value: "true",
|
|
|
|
Effect: "NoExecute",
|
|
|
|
},
|
|
|
|
}
|
2023-04-18 11:23:03 +00:00
|
|
|
BeforeEach(func(ctx context.Context) {
|
2023-04-05 17:42:20 +00:00
|
|
|
extraMasterPodSpecOpts = []testpod.SpecOption{
|
|
|
|
testpod.SpecWithContainerExtraArgs("-enable-taints"),
|
|
|
|
testpod.SpecWithTolerations(testTolerations),
|
|
|
|
}
|
|
|
|
})
|
2023-04-18 11:23:03 +00:00
|
|
|
It("custom labels from the NodeFeatureRule rules should be created", func(ctx context.Context) {
|
|
|
|
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
|
2023-02-25 11:42:39 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
targetNodeName := nodes[0].Name
|
|
|
|
Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
|
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
By("Creating nfd-worker config")
|
|
|
|
cm := testutils.NewConfigMap("nfd-worker-conf", "nfd-worker.conf", `
|
2022-11-29 08:19:11 +00:00
|
|
|
core:
|
|
|
|
sleepInterval: "1s"
|
|
|
|
featureSources: ["fake"]
|
|
|
|
labelSources: []
|
|
|
|
`)
|
2023-04-18 11:23:03 +00:00
|
|
|
cm, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
By("Creating nfd-worker daemonset")
|
2022-12-14 09:46:29 +00:00
|
|
|
podSpecOpts := createPodSpecOpts(
|
2023-01-11 10:14:43 +00:00
|
|
|
testpod.SpecWithContainerImage(dockerImage()),
|
2022-12-14 14:06:04 +00:00
|
|
|
testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"),
|
|
|
|
testpod.SpecWithTolerations(testTolerations),
|
2022-12-14 09:46:29 +00:00
|
|
|
)
|
2022-12-14 14:06:04 +00:00
|
|
|
workerDS := testds.NFDWorker(podSpecOpts...)
|
2023-04-18 11:23:03 +00:00
|
|
|
workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(ctx, workerDS, metav1.CreateOptions{})
|
2022-12-14 14:06:04 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
2023-03-07 09:11:52 +00:00
|
|
|
By("Waiting for worker daemonset pods to be ready")
|
2023-04-18 11:23:03 +00:00
|
|
|
Expect(testpod.WaitForReady(ctx, f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 2)).NotTo(HaveOccurred())
|
2022-12-14 14:06:04 +00:00
|
|
|
|
2023-04-20 15:09:50 +00:00
|
|
|
expectedLabels := map[string]k8sLabels{
|
2022-12-22 13:49:43 +00:00
|
|
|
"*": {
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/e2e-flag-test-1": "true",
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/e2e-attribute-test-1": "true",
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/e2e-instance-test-1": "true",
|
|
|
|
},
|
|
|
|
}
|
2022-12-14 14:06:04 +00:00
|
|
|
|
|
|
|
By("Creating NodeFeatureRules #1")
|
2023-04-18 11:23:03 +00:00
|
|
|
Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred())
|
2022-12-14 14:06:04 +00:00
|
|
|
|
|
|
|
By("Verifying node labels from NodeFeatureRules #1")
|
2023-04-20 15:09:50 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2022-12-14 14:06:04 +00:00
|
|
|
|
|
|
|
By("Creating NodeFeatureRules #2")
|
2023-04-18 11:23:03 +00:00
|
|
|
Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred())
|
2022-12-14 14:06:04 +00:00
|
|
|
|
|
|
|
// Add features from NodeFeatureRule #2
|
2023-04-20 15:09:50 +00:00
|
|
|
expectedLabels["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-matchany-test-1"] = "true"
|
|
|
|
expectedLabels["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-template-test-1-instance_1"] = "found"
|
|
|
|
expectedLabels["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-template-test-1-instance_2"] = "found"
|
2022-12-14 14:06:04 +00:00
|
|
|
|
|
|
|
By("Verifying node labels from NodeFeatureRules #1 and #2")
|
2023-04-20 15:09:50 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2022-12-14 14:06:04 +00:00
|
|
|
|
|
|
|
// Add features from NodeFeatureRule #3
|
|
|
|
By("Creating NodeFeatureRules #3")
|
2023-04-18 11:23:03 +00:00
|
|
|
Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-3.yaml")).NotTo(HaveOccurred())
|
2022-12-14 14:06:04 +00:00
|
|
|
|
|
|
|
By("Verifying node taints and annotation from NodeFeatureRules #3")
|
2023-04-21 14:04:39 +00:00
|
|
|
expectedTaints := map[string][]corev1.Taint{
|
|
|
|
"*": {
|
|
|
|
{
|
|
|
|
Key: "feature.node.kubernetes.io/fake-special-node",
|
|
|
|
Value: "exists",
|
|
|
|
Effect: "PreferNoSchedule",
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Key: "feature.node.kubernetes.io/fake-dedicated-node",
|
|
|
|
Value: "true",
|
|
|
|
Effect: "NoExecute",
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Key: "feature.node.kubernetes.io/performance-optimized-node",
|
|
|
|
Value: "true",
|
|
|
|
Effect: "NoExecute",
|
|
|
|
},
|
2022-12-14 14:06:04 +00:00
|
|
|
},
|
|
|
|
}
|
2023-04-21 09:57:27 +00:00
|
|
|
expectedAnnotations := map[string]k8sAnnotations{
|
|
|
|
"*": {
|
|
|
|
"nfd.node.kubernetes.io/taints": "feature.node.kubernetes.io/fake-special-node=exists:PreferNoSchedule,feature.node.kubernetes.io/fake-dedicated-node=true:NoExecute,feature.node.kubernetes.io/performance-optimized-node=true:NoExecute"},
|
|
|
|
}
|
2023-04-21 14:04:39 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes, false))
|
2023-04-21 09:57:27 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes, true))
|
2022-12-14 14:06:04 +00:00
|
|
|
|
|
|
|
By("Re-applying NodeFeatureRules #3 with updated taints")
|
2023-04-18 11:23:03 +00:00
|
|
|
Expect(testutils.UpdateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-3-updated.yaml")).NotTo(HaveOccurred())
|
2023-04-21 14:04:39 +00:00
|
|
|
expectedTaints["*"] = []corev1.Taint{
|
2022-12-14 14:06:04 +00:00
|
|
|
{
|
2023-04-05 12:27:50 +00:00
|
|
|
Key: "feature.node.kubernetes.io/fake-special-node",
|
2022-12-14 14:06:04 +00:00
|
|
|
Value: "exists",
|
|
|
|
Effect: "PreferNoSchedule",
|
|
|
|
},
|
|
|
|
{
|
2023-04-05 12:27:50 +00:00
|
|
|
Key: "feature.node.kubernetes.io/foo",
|
2022-12-14 14:06:04 +00:00
|
|
|
Value: "true",
|
|
|
|
Effect: "NoExecute",
|
|
|
|
},
|
|
|
|
}
|
2023-04-21 09:57:27 +00:00
|
|
|
expectedAnnotations["*"]["nfd.node.kubernetes.io/taints"] = "feature.node.kubernetes.io/fake-special-node=exists:PreferNoSchedule,feature.node.kubernetes.io/foo=true:NoExecute"
|
2022-11-30 23:21:21 +00:00
|
|
|
|
2022-12-14 14:06:04 +00:00
|
|
|
By("Verifying updated node taints and annotation from NodeFeatureRules #3")
|
2023-04-21 14:04:39 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes, false))
|
2023-04-21 09:57:27 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes, true))
|
2023-03-07 09:02:43 +00:00
|
|
|
|
2023-03-10 10:39:41 +00:00
|
|
|
By("Deleting NodeFeatureRule object")
|
2023-04-18 11:23:03 +00:00
|
|
|
err = nfdClient.NfdV1alpha1().NodeFeatureRules().Delete(ctx, "e2e-test-3", metav1.DeleteOptions{})
|
2023-03-10 10:39:41 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
2023-04-21 14:04:39 +00:00
|
|
|
expectedTaints["*"] = []corev1.Taint{}
|
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes, false))
|
2023-03-10 10:39:41 +00:00
|
|
|
|
2023-04-21 09:57:27 +00:00
|
|
|
expectedAnnotations["*"] = k8sAnnotations{"nfd.node.kubernetes.io/extended-resources": "nons,vendor.io/dynamic,vendor.io/static"}
|
2023-03-10 10:39:41 +00:00
|
|
|
|
2023-04-21 10:27:24 +00:00
|
|
|
expectedCapacity := map[string]corev1.ResourceList{
|
|
|
|
"*": {
|
|
|
|
"feature.node.kubernetes.io/nons": resourcev1.MustParse("123"),
|
|
|
|
"vendor.io/dynamic": resourcev1.MustParse("10"),
|
|
|
|
"vendor.io/static": resourcev1.MustParse("123"),
|
|
|
|
},
|
2023-03-10 10:39:41 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
By("Creating NodeFeatureRules #4")
|
2023-04-18 11:23:03 +00:00
|
|
|
Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-4.yaml")).NotTo(HaveOccurred())
|
2023-03-10 10:39:41 +00:00
|
|
|
|
|
|
|
By("Verifying node annotations from NodeFeatureRules #4")
|
2023-04-21 09:57:27 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes, true))
|
2023-03-10 10:39:41 +00:00
|
|
|
|
|
|
|
By("Verfiying node status capacity from NodeFeatureRules #4")
|
2023-04-21 10:27:24 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchCapacity(expectedCapacity, nodes, false))
|
2023-03-10 10:39:41 +00:00
|
|
|
|
|
|
|
By("Deleting NodeFeatureRule object")
|
2023-04-18 11:23:03 +00:00
|
|
|
err = nfdClient.NfdV1alpha1().NodeFeatureRules().Delete(ctx, "e2e-extened-resource-test", metav1.DeleteOptions{})
|
2023-03-10 10:39:41 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
By("Verfiying node status capacity from NodeFeatureRules #4")
|
2023-04-21 10:27:24 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchCapacity(expectedCapacity, nodes, false))
|
2023-03-10 10:39:41 +00:00
|
|
|
|
2023-03-07 09:02:43 +00:00
|
|
|
By("Deleting nfd-worker daemonset")
|
2023-04-18 11:23:03 +00:00
|
|
|
err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(ctx, workerDS.Name, metav1.DeleteOptions{})
|
2023-03-07 09:02:43 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
2022-12-14 14:06:04 +00:00
|
|
|
})
|
2022-07-07 07:40:19 +00:00
|
|
|
})
|
2023-03-05 21:56:46 +00:00
|
|
|
|
|
|
|
Context("and check whether master config passed successfully or not", func() {
|
2023-04-18 11:23:03 +00:00
|
|
|
BeforeEach(func(ctx context.Context) {
|
2023-04-05 17:42:20 +00:00
|
|
|
extraMasterPodSpecOpts = []testpod.SpecOption{
|
2023-03-05 21:56:46 +00:00
|
|
|
testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"),
|
|
|
|
}
|
|
|
|
cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", `
|
|
|
|
denyLabelNs: ["*.denied.ns","random.unwanted.ns"]
|
|
|
|
`)
|
2023-04-18 11:23:03 +00:00
|
|
|
_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
|
2023-03-05 21:56:46 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
})
|
2023-04-18 11:23:03 +00:00
|
|
|
It("master configuration should take place", func(ctx context.Context) {
|
2023-03-05 21:56:46 +00:00
|
|
|
// deploy node feature object
|
|
|
|
if !useNodeFeatureApi {
|
|
|
|
Skip("NodeFeature API not enabled")
|
|
|
|
}
|
|
|
|
|
2023-04-18 11:23:03 +00:00
|
|
|
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
|
2023-03-05 21:56:46 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
targetNodeName := nodes[0].Name
|
|
|
|
Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
|
|
|
|
|
|
|
|
// Apply Node Feature object
|
|
|
|
By("Create NodeFeature object")
|
2023-04-18 11:23:03 +00:00
|
|
|
nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(ctx, nfdClient, "nodefeature-3.yaml", f.Namespace.Name, targetNodeName)
|
2023-03-05 21:56:46 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
// Verify that denied label was not added
|
|
|
|
By("Verifying that denied labels were not added")
|
|
|
|
expectedLabels := map[string]k8sLabels{
|
|
|
|
targetNodeName: {
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-4": "obj-4",
|
|
|
|
"custom.vendor.io/e2e-nodefeature-test-3": "vendor-ns",
|
|
|
|
},
|
|
|
|
}
|
2023-04-20 15:09:50 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2023-03-05 21:56:46 +00:00
|
|
|
By("Deleting NodeFeature object")
|
2023-04-18 11:23:03 +00:00
|
|
|
err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{})
|
2023-03-05 21:56:46 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
// TODO: Find a better way to handle the timeout that happens to reflect the configmap changes
|
|
|
|
Skip("Testing the master dynamic configuration")
|
|
|
|
// Verify that config changes were applied
|
|
|
|
By("Updating the master config")
|
2023-04-18 11:23:03 +00:00
|
|
|
Expect(testutils.UpdateConfigMap(ctx, f.ClientSet, "nfd-master-conf", f.Namespace.Name, "nfd-master.conf", `
|
2023-03-05 21:56:46 +00:00
|
|
|
denyLabelNs: []
|
|
|
|
`))
|
|
|
|
By("Verifying that denied labels were removed")
|
|
|
|
expectedLabels = map[string]k8sLabels{
|
|
|
|
targetNodeName: {
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-4": "obj-4",
|
|
|
|
"custom.vendor.io/e2e-nodefeature-test-3": "vendor-ns",
|
|
|
|
"random.denied.ns/e2e-nodefeature-test-1": "denied-ns",
|
|
|
|
"random.unwanted.ns/e2e-nodefeature-test-2": "unwanted-ns",
|
|
|
|
},
|
|
|
|
}
|
2023-04-20 15:09:50 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2023-03-05 21:56:46 +00:00
|
|
|
})
|
|
|
|
})
|
2023-04-25 01:03:02 +00:00
|
|
|
|
|
|
|
Context("and test whether resyncPeriod is passed successfully or not", func() {
|
|
|
|
BeforeEach(func(ctx context.Context) {
|
|
|
|
extraMasterPodSpecOpts = []testpod.SpecOption{
|
|
|
|
testpod.SpecWithConfigMap("nfd-master-conf", "/etc/kubernetes/node-feature-discovery"),
|
|
|
|
}
|
|
|
|
cm := testutils.NewConfigMap("nfd-master-conf", "nfd-master.conf", `
|
|
|
|
resyncPeriod: "1s"
|
|
|
|
`)
|
|
|
|
_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
|
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
})
|
|
|
|
It("labels should be restored to the original ones", func(ctx context.Context) {
|
|
|
|
// deploy node feature object
|
|
|
|
if !useNodeFeatureApi {
|
|
|
|
Skip("NodeFeature API not enabled")
|
|
|
|
}
|
|
|
|
|
|
|
|
nodes, err := getNonControlPlaneNodes(ctx, f.ClientSet)
|
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
targetNodeName := nodes[0].Name
|
|
|
|
Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
|
|
|
|
|
|
|
|
// Apply Node Feature object
|
|
|
|
By("Creating NodeFeature object")
|
|
|
|
nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(ctx, nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName)
|
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
By("Verifying node labels from NodeFeature object #1")
|
|
|
|
expectedLabels := map[string]k8sLabels{
|
|
|
|
targetNodeName: {
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-1": "obj-1",
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-2": "obj-1",
|
|
|
|
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "overridden",
|
|
|
|
},
|
2023-04-20 15:09:50 +00:00
|
|
|
"*": {},
|
2023-04-25 01:03:02 +00:00
|
|
|
}
|
2023-04-20 15:09:50 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2023-04-25 01:03:02 +00:00
|
|
|
|
|
|
|
patches, err := json.Marshal(
|
|
|
|
[]apihelper.JsonPatch{
|
|
|
|
apihelper.NewJsonPatch(
|
|
|
|
"replace",
|
|
|
|
"/metadata/labels",
|
|
|
|
nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-1",
|
|
|
|
"randomValue",
|
|
|
|
),
|
|
|
|
},
|
|
|
|
)
|
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
|
|
|
_, err = f.ClientSet.CoreV1().Nodes().Patch(ctx, targetNodeName, types.JSONPatchType, patches, metav1.PatchOptions{})
|
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
|
2023-04-20 15:09:50 +00:00
|
|
|
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
|
2023-04-25 01:03:02 +00:00
|
|
|
|
|
|
|
By("Deleting NodeFeature object")
|
|
|
|
err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{})
|
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
})
|
|
|
|
})
|
2022-07-07 07:40:19 +00:00
|
|
|
})
|
2022-12-14 14:06:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Run the actual tests
|
|
|
|
Context("when running NFD with gRPC API enabled", func() {
|
|
|
|
nfdTestSuite(false)
|
2022-07-07 07:40:19 +00:00
|
|
|
})
|
2022-12-14 09:46:29 +00:00
|
|
|
|
|
|
|
Context("when running NFD with NodeFeature CRD API enabled", func() {
|
|
|
|
nfdTestSuite(true)
|
|
|
|
})
|
|
|
|
|
2019-04-25 14:41:26 +00:00
|
|
|
})
|
2022-07-07 07:40:19 +00:00
|
|
|
|
2022-12-16 20:10:36 +00:00
|
|
|
// getNonControlPlaneNodes gets the nodes that are not tainted for exclusive control-plane usage
|
2023-04-18 11:23:03 +00:00
|
|
|
func getNonControlPlaneNodes(ctx context.Context, cli clientset.Interface) ([]corev1.Node, error) {
|
|
|
|
nodeList, err := cli.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
|
2022-12-16 20:10:36 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if len(nodeList.Items) == 0 {
|
|
|
|
return nil, fmt.Errorf("no nodes found in the cluster")
|
|
|
|
}
|
|
|
|
|
|
|
|
controlPlaneTaint := corev1.Taint{
|
|
|
|
Effect: corev1.TaintEffectNoSchedule,
|
|
|
|
Key: "node-role.kubernetes.io/control-plane",
|
|
|
|
}
|
|
|
|
out := []corev1.Node{}
|
|
|
|
for _, node := range nodeList.Items {
|
|
|
|
if !taintutils.TaintExists(node.Spec.Taints, &controlPlaneTaint) {
|
|
|
|
out = append(out, node)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(out) == 0 {
|
|
|
|
return nil, fmt.Errorf("no non-control-plane nodes found in the cluster")
|
|
|
|
}
|
|
|
|
return out, nil
|
|
|
|
}
|
|
|
|
|
2023-04-05 12:27:50 +00:00
|
|
|
func getNode(nodes []corev1.Node, nodeName string) corev1.Node {
|
2023-02-25 11:42:39 +00:00
|
|
|
for _, node := range nodes {
|
|
|
|
if node.Name == nodeName {
|
2023-04-05 12:27:50 +00:00
|
|
|
return node
|
2022-07-07 07:40:19 +00:00
|
|
|
}
|
|
|
|
}
|
2023-04-05 12:27:50 +00:00
|
|
|
return corev1.Node{}
|
2022-07-07 07:40:19 +00:00
|
|
|
}
|