Merge pull request #842 from Garrybest/pr_config

nfd-topology-updater: retrieve kubelet config from API `/configz`

cmd/nfd-topology-updater/main.go

@@ -19,23 +19,27 @@ package main
 import (
 	"flag"
 	"fmt"
+	"net/url"
 	"os"
 	"time"
 
 	"k8s.io/klog/v2"
+	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
 
-	"sigs.k8s.io/node-feature-discovery/pkg/kubeconf"
+	nfdclient "sigs.k8s.io/node-feature-discovery/pkg/nfd-client"
 	topology "sigs.k8s.io/node-feature-discovery/pkg/nfd-client/topology-updater"
 	"sigs.k8s.io/node-feature-discovery/pkg/resourcemonitor"
 	"sigs.k8s.io/node-feature-discovery/pkg/topologypolicy"
 	"sigs.k8s.io/node-feature-discovery/pkg/utils"
 	"sigs.k8s.io/node-feature-discovery/pkg/utils/hostpath"
+	"sigs.k8s.io/node-feature-discovery/pkg/utils/kubeconf"
 	"sigs.k8s.io/node-feature-discovery/pkg/version"
 )
 
 const (
 	// ProgramName is the canonical name of this program
-	ProgramName = "nfd-topology-updater"
+	ProgramName       = "nfd-topology-updater"
+	kubeletSecurePort = 10250
 )
 
 func main() {
@@ -58,10 +62,33 @@ func main() {
 	// Plug klog into grpc logging infrastructure
 	utils.ConfigureGrpcKlog()
 
-	klConfig, err := kubeconf.GetKubeletConfigFromLocalFile(resourcemonitorArgs.KubeletConfigFile)
+	u, err := url.ParseRequestURI(resourcemonitorArgs.KubeletConfigURI)
 	if err != nil {
-		klog.Exitf("error reading kubelet config: %v", err)
+		klog.Exitf("failed to parse args for kubelet-config-uri: %v", err)
 	}
+
+	// init kubelet API client
+	var klConfig *kubeletconfigv1beta1.KubeletConfiguration
+	switch u.Scheme {
+	case "file":
+		klConfig, err = kubeconf.GetKubeletConfigFromLocalFile(u.Path)
+		if err != nil {
+			klog.Exitf("failed to read kubelet config: %v", err)
+		}
+	case "https":
+		restConfig, err := kubeconf.InsecureConfig(u.String(), resourcemonitorArgs.APIAuthTokenFile)
+		if err != nil {
+			klog.Exitf("failed to initialize rest config for kubelet config uri: %v", err)
+		}
+
+		klConfig, err = kubeconf.GetKubeletConfiguration(restConfig)
+		if err != nil {
+			klog.Exitf("failed to get kubelet config from configz endpoint: %v", err)
+		}
+	default:
+		klog.Exitf("unsupported URI scheme: %v", u.Scheme)
+	}
+
 	tmPolicy := string(topologypolicy.DetectTopologyPolicy(klConfig.TopologyManagerPolicy, klConfig.TopologyManagerScope))
 	klog.Infof("detected kubelet Topology Manager policy %q", tmPolicy)
 
@@ -86,6 +113,15 @@ func parseArgs(flags *flag.FlagSet, osArgs ...string) (*topology.Args, *resource
 		os.Exit(2)
 	}
 
+	if len(resourcemonitorArgs.KubeletConfigURI) == 0 {
+		if len(nfdclient.NodeName()) == 0 {
+			fmt.Fprintf(flags.Output(), "unable to determine the default kubelet config endpoint 'https://${NODE_NAME}:%d/configz' due to empty NODE_NAME environment, "+
+				"please either define the NODE_NAME environment variable or specify endpoint with the -kubelet-config-uri flag\n", kubeletSecurePort)
+			os.Exit(1)
+		}
+		resourcemonitorArgs.KubeletConfigURI = fmt.Sprintf("https://%s:%d/configz", nfdclient.NodeName(), kubeletSecurePort)
+	}
+
 	return args, resourcemonitorArgs
 }
 
@@ -109,8 +145,10 @@ func initFlags(flagset *flag.FlagSet) (*topology.Args, *resourcemonitor.Args) {
 		"Time to sleep between CR updates. Non-positive value implies no CR updatation (i.e. infinite sleep). [Default: 60s]")
 	flagset.StringVar(&resourcemonitorArgs.Namespace, "watch-namespace", "*",
 		"Namespace to watch pods (for testing/debugging purpose). Use * for all namespaces.")
-	flagset.StringVar(&resourcemonitorArgs.KubeletConfigFile, "kubelet-config-file", hostpath.VarDir.Path("lib/kubelet/config.yaml"),
+	flagset.StringVar(&resourcemonitorArgs.KubeletConfigURI, "kubelet-config-uri", "",
-		"Kubelet config file path.")
+		"Kubelet config URI path. Default to kubelet configz endpoint.")
+	flagset.StringVar(&resourcemonitorArgs.APIAuthTokenFile, "api-auth-token-file", "/var/run/secrets/kubernetes.io/serviceaccount/token",
+		"API auth token file path. It is used to request kubelet configz endpoint, only takes effect when kubelet-config-uri is https. Default to /var/run/secrets/kubernetes.io/serviceaccount/token.")
 	flagset.StringVar(&resourcemonitorArgs.PodResourceSocketPath, "podresources-socket", hostpath.VarDir.Path("lib/kubelet/pod-resources/kubelet.sock"),
 		"Pod Resource Socket path to use.")
 	flagset.StringVar(&args.Server, "server", "localhost:8080",

cmd/nfd-topology-updater/main_test.go

@@ -29,33 +29,33 @@ func TestArgsParse(t *testing.T) {
 		flags := flag.NewFlagSet(ProgramName, flag.ExitOnError)
 
 		Convey("When -no-publish and -oneshot flags are passed", func() {
-			args, finderArgs := parseArgs(flags, "-oneshot", "-no-publish")
+			args, finderArgs := parseArgs(flags, "-oneshot", "-no-publish", "-kubelet-config-uri=https://%s:%d/configz")
 
 			Convey("noPublish is set and args.sources is set to the default value", func() {
 				So(args.NoPublish, ShouldBeTrue)
 				So(args.Oneshot, ShouldBeTrue)
 				So(finderArgs.SleepInterval, ShouldEqual, 60*time.Second)
-				So(finderArgs.KubeletConfigFile, ShouldEqual, "/var/lib/kubelet/config.yaml")
 				So(finderArgs.PodResourceSocketPath, ShouldEqual, "/var/lib/kubelet/pod-resources/kubelet.sock")
 			})
 		})
 
-		Convey("When valid args are specified for -kubelet-config-file and -sleep-interval,", func() {
+		Convey("When valid args are specified for -kubelet-config-url and -sleep-interval,", func() {
 			args, finderArgs := parseArgs(flags,
-				"-kubelet-config-file=/path/testconfig.yaml",
+				"-kubelet-config-uri=file:///path/testconfig.yaml",
 				"-sleep-interval=30s")
 
 			Convey("args.sources is set to appropriate values", func() {
 				So(args.NoPublish, ShouldBeFalse)
 				So(args.Oneshot, ShouldBeFalse)
 				So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second)
-				So(finderArgs.KubeletConfigFile, ShouldEqual, "/path/testconfig.yaml")
+				So(finderArgs.KubeletConfigURI, ShouldEqual, "file:///path/testconfig.yaml")
 				So(finderArgs.PodResourceSocketPath, ShouldEqual, "/var/lib/kubelet/pod-resources/kubelet.sock")
 			})
 		})
 
 		Convey("When valid args are specified for -podresources-socket flag and -sleep-interval is specified", func() {
 			args, finderArgs := parseArgs(flags,
+				"-kubelet-config-uri=https://%s:%d/configz",
 				"-podresources-socket=/path/testkubelet.sock",
 				"-sleep-interval=30s")
 
@@ -63,19 +63,18 @@ func TestArgsParse(t *testing.T) {
 				So(args.NoPublish, ShouldBeFalse)
 				So(args.Oneshot, ShouldBeFalse)
 				So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second)
-				So(finderArgs.KubeletConfigFile, ShouldEqual, "/var/lib/kubelet/config.yaml")
 				So(finderArgs.PodResourceSocketPath, ShouldEqual, "/path/testkubelet.sock")
 			})
 		})
 		Convey("When valid -sleep-inteval is specified", func() {
 			args, finderArgs := parseArgs(flags,
+				"-kubelet-config-uri=https://%s:%d/configz",
 				"-sleep-interval=30s")
 
 			Convey("args.sources is set to appropriate values", func() {
 				So(args.NoPublish, ShouldBeFalse)
 				So(args.Oneshot, ShouldBeFalse)
 				So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second)
-				So(finderArgs.KubeletConfigFile, ShouldEqual, "/var/lib/kubelet/config.yaml")
 				So(finderArgs.PodResourceSocketPath, ShouldEqual, "/var/lib/kubelet/pod-resources/kubelet.sock")
 			})
 		})
@@ -84,7 +83,7 @@ func TestArgsParse(t *testing.T) {
 			args, finderArgs := parseArgs(flags,
 				"-no-publish",
 				"-sleep-interval=30s",
-				"-kubelet-config-file=/path/testconfig.yaml",
+				"-kubelet-config-uri=file:///path/testconfig.yaml",
 				"-podresources-socket=/path/testkubelet.sock",
 				"-ca-file=ca",
 				"-cert-file=crt",
@@ -96,7 +95,7 @@ func TestArgsParse(t *testing.T) {
 				So(args.CertFile, ShouldEqual, "crt")
 				So(args.KeyFile, ShouldEqual, "key")
 				So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second)
-				So(finderArgs.KubeletConfigFile, ShouldEqual, "/path/testconfig.yaml")
+				So(finderArgs.KubeletConfigURI, ShouldEqual, "file:///path/testconfig.yaml")
 				So(finderArgs.PodResourceSocketPath, ShouldEqual, "/path/testkubelet.sock")
 			})
 		})

deployment/base/rbac-topologyupdater/topologyupdater-clusterrole.yaml

@@ -10,6 +10,12 @@ rules:
   verbs:
   - get
   - list
+- apiGroups:
+  - ""
+  resources:
+  - nodes/proxy
+  verbs:
+  - get
 - apiGroups:
   - ""
   resources:

deployment/components/topology-updater/topologyupdater-mounts.yaml

@@ -21,10 +21,6 @@
   - name: host-sys
     mountPath: /host-sys
 
-- op: add
-  path: /spec/template/spec/containers/0/args/-
-  value: "-kubelet-config-file=/host-var/lib/kubelet/config.yaml"
-
 - op: add
   path: /spec/template/spec/containers/0/args/-
   value: "-podresources-socket=/host-var/lib/kubelet/pod-resources/kubelet.sock"

deployment/helm/node-feature-discovery/templates/clusterrole.yaml

@@ -18,6 +18,12 @@ rules:
   - patch
   - update
   - list
+- apiGroups:
+    - ""
+  resources:
+    - nodes/proxy
+  verbs:
+    - get
 - apiGroups:
   - nfd.k8s-sigs.io
   resources:

docs/reference/topology-updater-commandline-reference.md

@@ -169,17 +169,33 @@ Example:
 nfd-topology-updater -watch-namespace=rte
 ```
 
-### -kubelet-config-file
+### -kubelet-config-uri
 
-The `-kubelet-config-file` specifies the path to the Kubelet's configuration
+The `-kubelet-config-uri` specifies the path to the Kubelet's configuration.
-file.
+Note that the URi could either be a local host file or an HTTP endpoint.
 
-Default:  /host-var/lib/kubelet/config.yaml
+Default:  `https://${NODE_NAME}:10250/configz`
 
 Example:
 
 ```bash
-nfd-topology-updater -kubelet-config-file=/var/lib/kubelet/config.yaml
+nfd-topology-updater -kubelet-config-uri=file:///var/lib/kubelet/config.yaml
+```
+
+### -api-auth-token-file
+
+The `-api-auth-token-file` specifies the path to the api auth token file
+which is used to retrieve Kubelet's configuration from Kubelet secure port,
+only taking effect when `-kubelet-config-uri` is https.
+Note that this token file must bind to a role that has the `get` capability to
+`nodes/proxy` resources.
+
+Default:  `/var/run/secrets/kubernetes.io/serviceaccount/token`
+
+Example:
+
+```bash
+nfd-topology-updater -token-file=/var/run/secrets/kubernetes.io/serviceaccount/token
 ```
 
 ### -podresources-socket

pkg/resourcemonitor/types.go

@@ -29,7 +29,8 @@ type Args struct {
 	PodResourceSocketPath string
 	SleepInterval         time.Duration
 	Namespace             string
-	KubeletConfigFile     string
+	KubeletConfigURI      string
+	APIAuthTokenFile      string
 }
 
 // ResourceInfo stores information of resources and their corresponding IDs obtained from PodResource API

pkg/utils/kubeconf/kubelet_config_file_test.go

@@ -29,7 +29,7 @@ type testCaseData struct {
 func TestGetKubeletConfigFromLocalFile(t *testing.T) {
 	tCases := []testCaseData{
 		{
-			path:     filepath.Join("..", "..", "test", "data", "kubeletconf.yaml"),
+			path:     filepath.Join("..", "..", "..", "test", "data", "kubeletconf.yaml"),
 			tmPolicy: "single-numa-node",
 		},
 	}

pkg/utils/kubeconf/kubelet_configz.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubeconf
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"time"
+
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/rest"
+	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
+)
+
+// GetKubeletConfiguration returns the kubelet configuration.
+func GetKubeletConfiguration(restConfig *rest.Config) (*kubeletconfigv1beta1.KubeletConfiguration, error) {
+	discoveryClient, err := discovery.NewDiscoveryClientForConfig(restConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	var timeout time.Duration
+	// This hack because /configz reports the following structure:
+	// {"kubeletconfig": {the JSON representation of kubeletconfigv1beta1.KubeletConfiguration}}
+	type configzWrapper struct {
+		ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"`
+	}
+	bytes, err := discoveryClient.RESTClient().
+		Get().
+		Timeout(timeout).
+		Do(context.TODO()).
+		Raw()
+	if err != nil {
+		return nil, err
+	}
+
+	configz := configzWrapper{}
+	if err = json.Unmarshal(bytes, &configz); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal json for kubelet config: %w", err)
+	}
+
+	return &configz.ComponentConfig, nil
+}
+
+// InsecureConfig returns a kubelet API config object which uses the token path.
+func InsecureConfig(host, tokenFile string) (*rest.Config, error) {
+	if tokenFile == "" {
+		return nil, fmt.Errorf("api auth token file must be defined")
+	}
+	if len(host) == 0 {
+		return nil, fmt.Errorf("kubelet host must be defined")
+	}
+
+	token, err := os.ReadFile(tokenFile)
+	if err != nil {
+		return nil, err
+	}
+
+	tlsClientConfig := rest.TLSClientConfig{Insecure: true}
+
+	return &rest.Config{
+		Host:            host,
+		TLSClientConfig: tlsClientConfig,
+		BearerToken:     string(token),
+		BearerTokenFile: tokenFile,
+	}, nil
+}