diff --git a/cmd/nfd-topology-updater/main.go b/cmd/nfd-topology-updater/main.go index 0f8872ef7..bdb9b7c92 100644 --- a/cmd/nfd-topology-updater/main.go +++ b/cmd/nfd-topology-updater/main.go @@ -19,23 +19,27 @@ package main import ( "flag" "fmt" + "net/url" "os" "time" "k8s.io/klog/v2" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" - "sigs.k8s.io/node-feature-discovery/pkg/kubeconf" + nfdclient "sigs.k8s.io/node-feature-discovery/pkg/nfd-client" topology "sigs.k8s.io/node-feature-discovery/pkg/nfd-client/topology-updater" "sigs.k8s.io/node-feature-discovery/pkg/resourcemonitor" "sigs.k8s.io/node-feature-discovery/pkg/topologypolicy" "sigs.k8s.io/node-feature-discovery/pkg/utils" "sigs.k8s.io/node-feature-discovery/pkg/utils/hostpath" + "sigs.k8s.io/node-feature-discovery/pkg/utils/kubeconf" "sigs.k8s.io/node-feature-discovery/pkg/version" ) const ( // ProgramName is the canonical name of this program - ProgramName = "nfd-topology-updater" + ProgramName = "nfd-topology-updater" + kubeletSecurePort = 10250 ) func main() { @@ -58,10 +62,33 @@ func main() { // Plug klog into grpc logging infrastructure utils.ConfigureGrpcKlog() - klConfig, err := kubeconf.GetKubeletConfigFromLocalFile(resourcemonitorArgs.KubeletConfigFile) + u, err := url.ParseRequestURI(resourcemonitorArgs.KubeletConfigURI) if err != nil { - klog.Exitf("error reading kubelet config: %v", err) + klog.Exitf("failed to parse args for kubelet-config-uri: %v", err) } + + // init kubelet API client + var klConfig *kubeletconfigv1beta1.KubeletConfiguration + switch u.Scheme { + case "file": + klConfig, err = kubeconf.GetKubeletConfigFromLocalFile(u.Path) + if err != nil { + klog.Exitf("failed to read kubelet config: %v", err) + } + case "https": + restConfig, err := kubeconf.InsecureConfig(u.String(), resourcemonitorArgs.APIAuthTokenFile) + if err != nil { + klog.Exitf("failed to initialize rest config for kubelet config uri: %v", err) + } + + klConfig, err = kubeconf.GetKubeletConfiguration(restConfig) + if err != nil { + klog.Exitf("failed to get kubelet config from configz endpoint: %v", err) + } + default: + klog.Exitf("unsupported URI scheme: %v", u.Scheme) + } + tmPolicy := string(topologypolicy.DetectTopologyPolicy(klConfig.TopologyManagerPolicy, klConfig.TopologyManagerScope)) klog.Infof("detected kubelet Topology Manager policy %q", tmPolicy) @@ -86,6 +113,15 @@ func parseArgs(flags *flag.FlagSet, osArgs ...string) (*topology.Args, *resource os.Exit(2) } + if len(resourcemonitorArgs.KubeletConfigURI) == 0 { + if len(nfdclient.NodeName()) == 0 { + fmt.Fprintf(flags.Output(), "unable to determine the default kubelet config endpoint 'https://${NODE_NAME}:%d/configz' due to empty NODE_NAME environment, "+ + "please either define the NODE_NAME environment variable or specify endpoint with the -kubelet-config-uri flag\n", kubeletSecurePort) + os.Exit(1) + } + resourcemonitorArgs.KubeletConfigURI = fmt.Sprintf("https://%s:%d/configz", nfdclient.NodeName(), kubeletSecurePort) + } + return args, resourcemonitorArgs } @@ -109,8 +145,10 @@ func initFlags(flagset *flag.FlagSet) (*topology.Args, *resourcemonitor.Args) { "Time to sleep between CR updates. Non-positive value implies no CR updatation (i.e. infinite sleep). [Default: 60s]") flagset.StringVar(&resourcemonitorArgs.Namespace, "watch-namespace", "*", "Namespace to watch pods (for testing/debugging purpose). Use * for all namespaces.") - flagset.StringVar(&resourcemonitorArgs.KubeletConfigFile, "kubelet-config-file", hostpath.VarDir.Path("lib/kubelet/config.yaml"), - "Kubelet config file path.") + flagset.StringVar(&resourcemonitorArgs.KubeletConfigURI, "kubelet-config-uri", "", + "Kubelet config URI path. Default to kubelet configz endpoint.") + flagset.StringVar(&resourcemonitorArgs.APIAuthTokenFile, "api-auth-token-file", "/var/run/secrets/kubernetes.io/serviceaccount/token", + "API auth token file path. It is used to request kubelet configz endpoint, only takes effect when kubelet-config-uri is https. Default to /var/run/secrets/kubernetes.io/serviceaccount/token.") flagset.StringVar(&resourcemonitorArgs.PodResourceSocketPath, "podresources-socket", hostpath.VarDir.Path("lib/kubelet/pod-resources/kubelet.sock"), "Pod Resource Socket path to use.") flagset.StringVar(&args.Server, "server", "localhost:8080", diff --git a/cmd/nfd-topology-updater/main_test.go b/cmd/nfd-topology-updater/main_test.go index 9094bb023..e1b249808 100644 --- a/cmd/nfd-topology-updater/main_test.go +++ b/cmd/nfd-topology-updater/main_test.go @@ -29,33 +29,33 @@ func TestArgsParse(t *testing.T) { flags := flag.NewFlagSet(ProgramName, flag.ExitOnError) Convey("When -no-publish and -oneshot flags are passed", func() { - args, finderArgs := parseArgs(flags, "-oneshot", "-no-publish") + args, finderArgs := parseArgs(flags, "-oneshot", "-no-publish", "-kubelet-config-uri=https://%s:%d/configz") Convey("noPublish is set and args.sources is set to the default value", func() { So(args.NoPublish, ShouldBeTrue) So(args.Oneshot, ShouldBeTrue) So(finderArgs.SleepInterval, ShouldEqual, 60*time.Second) - So(finderArgs.KubeletConfigFile, ShouldEqual, "/var/lib/kubelet/config.yaml") So(finderArgs.PodResourceSocketPath, ShouldEqual, "/var/lib/kubelet/pod-resources/kubelet.sock") }) }) - Convey("When valid args are specified for -kubelet-config-file and -sleep-interval,", func() { + Convey("When valid args are specified for -kubelet-config-url and -sleep-interval,", func() { args, finderArgs := parseArgs(flags, - "-kubelet-config-file=/path/testconfig.yaml", + "-kubelet-config-uri=file:///path/testconfig.yaml", "-sleep-interval=30s") Convey("args.sources is set to appropriate values", func() { So(args.NoPublish, ShouldBeFalse) So(args.Oneshot, ShouldBeFalse) So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second) - So(finderArgs.KubeletConfigFile, ShouldEqual, "/path/testconfig.yaml") + So(finderArgs.KubeletConfigURI, ShouldEqual, "file:///path/testconfig.yaml") So(finderArgs.PodResourceSocketPath, ShouldEqual, "/var/lib/kubelet/pod-resources/kubelet.sock") }) }) Convey("When valid args are specified for -podresources-socket flag and -sleep-interval is specified", func() { args, finderArgs := parseArgs(flags, + "-kubelet-config-uri=https://%s:%d/configz", "-podresources-socket=/path/testkubelet.sock", "-sleep-interval=30s") @@ -63,19 +63,18 @@ func TestArgsParse(t *testing.T) { So(args.NoPublish, ShouldBeFalse) So(args.Oneshot, ShouldBeFalse) So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second) - So(finderArgs.KubeletConfigFile, ShouldEqual, "/var/lib/kubelet/config.yaml") So(finderArgs.PodResourceSocketPath, ShouldEqual, "/path/testkubelet.sock") }) }) Convey("When valid -sleep-inteval is specified", func() { args, finderArgs := parseArgs(flags, + "-kubelet-config-uri=https://%s:%d/configz", "-sleep-interval=30s") Convey("args.sources is set to appropriate values", func() { So(args.NoPublish, ShouldBeFalse) So(args.Oneshot, ShouldBeFalse) So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second) - So(finderArgs.KubeletConfigFile, ShouldEqual, "/var/lib/kubelet/config.yaml") So(finderArgs.PodResourceSocketPath, ShouldEqual, "/var/lib/kubelet/pod-resources/kubelet.sock") }) }) @@ -84,7 +83,7 @@ func TestArgsParse(t *testing.T) { args, finderArgs := parseArgs(flags, "-no-publish", "-sleep-interval=30s", - "-kubelet-config-file=/path/testconfig.yaml", + "-kubelet-config-uri=file:///path/testconfig.yaml", "-podresources-socket=/path/testkubelet.sock", "-ca-file=ca", "-cert-file=crt", @@ -96,7 +95,7 @@ func TestArgsParse(t *testing.T) { So(args.CertFile, ShouldEqual, "crt") So(args.KeyFile, ShouldEqual, "key") So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second) - So(finderArgs.KubeletConfigFile, ShouldEqual, "/path/testconfig.yaml") + So(finderArgs.KubeletConfigURI, ShouldEqual, "file:///path/testconfig.yaml") So(finderArgs.PodResourceSocketPath, ShouldEqual, "/path/testkubelet.sock") }) }) diff --git a/deployment/base/rbac-topologyupdater/topologyupdater-clusterrole.yaml b/deployment/base/rbac-topologyupdater/topologyupdater-clusterrole.yaml index 35de71b10..1719edbe0 100644 --- a/deployment/base/rbac-topologyupdater/topologyupdater-clusterrole.yaml +++ b/deployment/base/rbac-topologyupdater/topologyupdater-clusterrole.yaml @@ -10,6 +10,12 @@ rules: verbs: - get - list +- apiGroups: + - "" + resources: + - nodes/proxy + verbs: + - get - apiGroups: - "" resources: diff --git a/deployment/components/topology-updater/topologyupdater-mounts.yaml b/deployment/components/topology-updater/topologyupdater-mounts.yaml index c792c99e9..89f1c2e5e 100644 --- a/deployment/components/topology-updater/topologyupdater-mounts.yaml +++ b/deployment/components/topology-updater/topologyupdater-mounts.yaml @@ -21,10 +21,6 @@ - name: host-sys mountPath: /host-sys -- op: add - path: /spec/template/spec/containers/0/args/- - value: "-kubelet-config-file=/host-var/lib/kubelet/config.yaml" - - op: add path: /spec/template/spec/containers/0/args/- value: "-podresources-socket=/host-var/lib/kubelet/pod-resources/kubelet.sock" diff --git a/deployment/helm/node-feature-discovery/templates/clusterrole.yaml b/deployment/helm/node-feature-discovery/templates/clusterrole.yaml index 36a12ecbe..587a5a955 100644 --- a/deployment/helm/node-feature-discovery/templates/clusterrole.yaml +++ b/deployment/helm/node-feature-discovery/templates/clusterrole.yaml @@ -18,6 +18,12 @@ rules: - patch - update - list +- apiGroups: + - "" + resources: + - nodes/proxy + verbs: + - get - apiGroups: - nfd.k8s-sigs.io resources: diff --git a/docs/reference/topology-updater-commandline-reference.md b/docs/reference/topology-updater-commandline-reference.md index e872e8d85..742d36c66 100644 --- a/docs/reference/topology-updater-commandline-reference.md +++ b/docs/reference/topology-updater-commandline-reference.md @@ -169,17 +169,33 @@ Example: nfd-topology-updater -watch-namespace=rte ``` -### -kubelet-config-file +### -kubelet-config-uri -The `-kubelet-config-file` specifies the path to the Kubelet's configuration -file. +The `-kubelet-config-uri` specifies the path to the Kubelet's configuration. +Note that the URi could either be a local host file or an HTTP endpoint. -Default: /host-var/lib/kubelet/config.yaml +Default: `https://${NODE_NAME}:10250/configz` Example: ```bash -nfd-topology-updater -kubelet-config-file=/var/lib/kubelet/config.yaml +nfd-topology-updater -kubelet-config-uri=file:///var/lib/kubelet/config.yaml +``` + +### -api-auth-token-file + +The `-api-auth-token-file` specifies the path to the api auth token file +which is used to retrieve Kubelet's configuration from Kubelet secure port, +only taking effect when `-kubelet-config-uri` is https. +Note that this token file must bind to a role that has the `get` capability to +`nodes/proxy` resources. + +Default: `/var/run/secrets/kubernetes.io/serviceaccount/token` + +Example: + +```bash +nfd-topology-updater -token-file=/var/run/secrets/kubernetes.io/serviceaccount/token ``` ### -podresources-socket diff --git a/pkg/resourcemonitor/types.go b/pkg/resourcemonitor/types.go index b288ef8c7..5b0a4c5fe 100644 --- a/pkg/resourcemonitor/types.go +++ b/pkg/resourcemonitor/types.go @@ -29,7 +29,8 @@ type Args struct { PodResourceSocketPath string SleepInterval time.Duration Namespace string - KubeletConfigFile string + KubeletConfigURI string + APIAuthTokenFile string } // ResourceInfo stores information of resources and their corresponding IDs obtained from PodResource API diff --git a/pkg/kubeconf/kubelet_config_file.go b/pkg/utils/kubeconf/kubelet_config_file.go similarity index 100% rename from pkg/kubeconf/kubelet_config_file.go rename to pkg/utils/kubeconf/kubelet_config_file.go diff --git a/pkg/kubeconf/kubelet_config_file_test.go b/pkg/utils/kubeconf/kubelet_config_file_test.go similarity index 93% rename from pkg/kubeconf/kubelet_config_file_test.go rename to pkg/utils/kubeconf/kubelet_config_file_test.go index e2891a4a0..622056117 100644 --- a/pkg/kubeconf/kubelet_config_file_test.go +++ b/pkg/utils/kubeconf/kubelet_config_file_test.go @@ -29,7 +29,7 @@ type testCaseData struct { func TestGetKubeletConfigFromLocalFile(t *testing.T) { tCases := []testCaseData{ { - path: filepath.Join("..", "..", "test", "data", "kubeletconf.yaml"), + path: filepath.Join("..", "..", "..", "test", "data", "kubeletconf.yaml"), tmPolicy: "single-numa-node", }, } diff --git a/pkg/utils/kubeconf/kubelet_configz.go b/pkg/utils/kubeconf/kubelet_configz.go new file mode 100644 index 000000000..a7791f5de --- /dev/null +++ b/pkg/utils/kubeconf/kubelet_configz.go @@ -0,0 +1,83 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeconf + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" +) + +// GetKubeletConfiguration returns the kubelet configuration. +func GetKubeletConfiguration(restConfig *rest.Config) (*kubeletconfigv1beta1.KubeletConfiguration, error) { + discoveryClient, err := discovery.NewDiscoveryClientForConfig(restConfig) + if err != nil { + return nil, err + } + + var timeout time.Duration + // This hack because /configz reports the following structure: + // {"kubeletconfig": {the JSON representation of kubeletconfigv1beta1.KubeletConfiguration}} + type configzWrapper struct { + ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"` + } + bytes, err := discoveryClient.RESTClient(). + Get(). + Timeout(timeout). + Do(context.TODO()). + Raw() + if err != nil { + return nil, err + } + + configz := configzWrapper{} + if err = json.Unmarshal(bytes, &configz); err != nil { + return nil, fmt.Errorf("failed to unmarshal json for kubelet config: %w", err) + } + + return &configz.ComponentConfig, nil +} + +// InsecureConfig returns a kubelet API config object which uses the token path. +func InsecureConfig(host, tokenFile string) (*rest.Config, error) { + if tokenFile == "" { + return nil, fmt.Errorf("api auth token file must be defined") + } + if len(host) == 0 { + return nil, fmt.Errorf("kubelet host must be defined") + } + + token, err := os.ReadFile(tokenFile) + if err != nil { + return nil, err + } + + tlsClientConfig := rest.TLSClientConfig{Insecure: true} + + return &rest.Config{ + Host: host, + TLSClientConfig: tlsClientConfig, + BearerToken: string(token), + BearerTokenFile: tokenFile, + }, nil +}