From 39be798472d86be6fcffd7a2341a151f21152374 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Fri, 11 Jan 2019 15:55:28 +0200 Subject: [PATCH] Split NFD into client and server Refactor NFD into a simple server-client system. Labeling is now done by a separate 'nfd-master' server. It is a simple service with small codebase, designed for easy isolation. The feature discovery part is implemented in a 'nfd-worker' client which sends labeling requests to nfd-server, thus, requiring no access/permissions to the Kubernetes API itself. Client-server communication is implemented by using gRPC. The protocol currently consists of only one request, i.e. the labeling request. The spec templates are converted to the new scheme. The nfd-master server can be deployed using the nfd-master.yaml.template which now also contains the necessary RBAC configuration. NFD workers can be deployed by using the nfd-worker-daemonset.yaml.template or nfd-worker-job.yaml.template (most easily used with the label-nodes.sh script). Only nfd-worker currently support config file or options. The (default) NFD config file is renamed to nfd-worker.conf. --- Dockerfile | 10 +- Gopkg.lock | 55 +++- Gopkg.toml | 8 + cmd/nfd-master/main.go | 246 ++++++++++++++++++ main.go => cmd/nfd-worker/main.go | 237 +++++------------ label-nodes.sh | 2 +- nfd-master.yaml.template | 78 ++++++ ...late => nfd-worker-daemonset.yaml.template | 14 +- ...l.template => nfd-worker-job.yaml.template | 8 +- ...ry.conf.example => nfd-worker.conf.example | 0 pkg/apihelper/apihelpers.go | 50 ++++ pkg/apihelper/k8shelpers.go | 95 +++++++ pkg/labeler/labeler.pb.go | 206 +++++++++++++++ pkg/labeler/labeler.proto | 35 +++ rbac.yaml | 34 --- 15 files changed, 851 insertions(+), 227 deletions(-) create mode 100644 cmd/nfd-master/main.go rename main.go => cmd/nfd-worker/main.go (63%) create mode 100644 nfd-master.yaml.template rename node-feature-discovery-daemonset.yaml.template => nfd-worker-daemonset.yaml.template (80%) rename node-feature-discovery-job.yaml.template => nfd-worker-job.yaml.template (87%) rename node-feature-discovery.conf.example => nfd-worker.conf.example (100%) create mode 100644 pkg/apihelper/apihelpers.go create mode 100644 pkg/apihelper/k8shelpers.go create mode 100644 pkg/labeler/labeler.pb.go create mode 100644 pkg/labeler/labeler.proto delete mode 100644 rbac.yaml diff --git a/Dockerfile b/Dockerfile index c7e82d146..114907bd5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,16 +11,14 @@ RUN go get github.com/golang/dep/cmd/dep RUN dep ensure RUN go install \ -ldflags "-s -w -X sigs.k8s.io/node-feature-discovery/pkg/version.version=$NFD_VERSION" \ - sigs.k8s.io/node-feature-discovery -RUN install -D -m644 node-feature-discovery.conf.example /etc/kubernetes/node-feature-discovery/node-feature-discovery.conf + ./cmd/* +RUN install -D -m644 nfd-worker.conf.example /etc/kubernetes/node-feature-discovery/nfd-worker.conf -RUN go test . +#RUN go test . # Create production image for running node feature discovery FROM debian:stretch-slim COPY --from=builder /etc/kubernetes/node-feature-discovery /etc/kubernetes/node-feature-discovery -COPY --from=builder /go/bin/node-feature-discovery /usr/bin/node-feature-discovery - -ENTRYPOINT ["/usr/bin/node-feature-discovery"] +COPY --from=builder /go/bin/nfd-* /usr/bin/ diff --git a/Gopkg.lock b/Gopkg.lock index 49d6a126c..f51cd02cf 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -230,7 +230,7 @@ [[projects]] branch = "master" - digest = "1:37f4a0712bbf9b111cf2951fe0efeea300ce87f7915047d4d1d5806a256ca634" + digest = "1:470efb06ada11351d90ee09868d84c622cc949a59c165b99d33d555dacbde74b" name = "golang.org/x/net" packages = [ "context", @@ -239,6 +239,8 @@ "http2", "http2/hpack", "idna", + "internal/timeseries", + "trace", ] pruneopts = "UT" revision = "915654e7eabcea33ae277abbecf52f0d8b7a9fdc" @@ -312,6 +314,54 @@ revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1" version = "v1.4.0" +[[projects]] + branch = "master" + digest = "1:077c1c599507b3b3e9156d17d36e1e61928ee9b53a5b420f10f28ebd4a0b275c" + name = "google.golang.org/genproto" + packages = ["googleapis/rpc/status"] + pruneopts = "UT" + revision = "6909d8a4a91b6d3fd1c4580b6e35816be4706fef" + +[[projects]] + digest = "1:9edd250a3c46675d0679d87540b30c9ed253b19bd1fd1af08f4f5fb3c79fc487" + name = "google.golang.org/grpc" + packages = [ + ".", + "balancer", + "balancer/base", + "balancer/roundrobin", + "binarylog/grpc_binarylog_v1", + "codes", + "connectivity", + "credentials", + "credentials/internal", + "encoding", + "encoding/proto", + "grpclog", + "internal", + "internal/backoff", + "internal/binarylog", + "internal/channelz", + "internal/envconfig", + "internal/grpcrand", + "internal/grpcsync", + "internal/syscall", + "internal/transport", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "resolver/dns", + "resolver/passthrough", + "stats", + "status", + "tap", + ] + pruneopts = "UT" + revision = "df014850f6dee74ba2fc94874043a9f3f75fbfd8" + version = "v1.17.0" + [[projects]] digest = "1:2d1fbdc6777e5408cabeb02bf336305e724b925ff4546ded0fa8715a7267922a" name = "gopkg.in/inf.v0" @@ -495,10 +545,13 @@ "github.com/docopt/docopt-go", "github.com/ghodss/yaml", "github.com/golang/glog", + "github.com/golang/protobuf/proto", "github.com/klauspost/cpuid", "github.com/smartystreets/goconvey/convey", "github.com/stretchr/testify/mock", "github.com/vektra/errors", + "golang.org/x/net/context", + "google.golang.org/grpc", "k8s.io/api/core/v1", "k8s.io/apimachinery/pkg/apis/meta/v1", "k8s.io/client-go/kubernetes", diff --git a/Gopkg.toml b/Gopkg.toml index 002973967..9e13a5369 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -59,3 +59,11 @@ [prune] go-tests = true unused-packages = true + +[[constraint]] + name = "github.com/golang/protobuf" + version = "1.2.0" + +[[constraint]] + name = "google.golang.org/grpc" + version = "1.17.0" diff --git a/cmd/nfd-master/main.go b/cmd/nfd-master/main.go new file mode 100644 index 000000000..c2369309c --- /dev/null +++ b/cmd/nfd-master/main.go @@ -0,0 +1,246 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "log" + "net" + "os" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/docopt/docopt-go" + "golang.org/x/net/context" + "google.golang.org/grpc" + "sigs.k8s.io/node-feature-discovery/pkg/apihelper" + pb "sigs.k8s.io/node-feature-discovery/pkg/labeler" + "sigs.k8s.io/node-feature-discovery/pkg/version" +) + +const ( + // ProgramName is the canonical name of this program + ProgramName = "nfd-master" + + // NodeNameEnv is the environment variable that contains this node's name. + NodeNameEnv = "NODE_NAME" + + // Namespace for feature labels + labelNs = "feature.node.kubernetes.io/" + + // Namespace for all NFD-related annotations + annotationNs = "nfd.node.kubernetes.io/" +) + +// package loggers +var ( + stdoutLogger = log.New(os.Stdout, "", log.LstdFlags) + stderrLogger = log.New(os.Stderr, "", log.LstdFlags) +) + +// Labels are a Kubernetes representation of discovered features. +type Labels map[string]string + +// Annotations are used for NFD-related node metadata +type Annotations map[string]string + +// Command line arguments +type Args struct { + labelWhiteList *regexp.Regexp + noPublish bool + port int +} + +func main() { + // Assert that the version is known + if version.Get() == "undefined" { + stderrLogger.Fatalf("version not set! Set -ldflags \"-X sigs.k8s.io/node-feature-discovery/pkg/version.version=`git describe --tags --dirty --always`\" during build or run.") + } + stdoutLogger.Printf("Node Feature Discovery Master %s", version.Get()) + + // Parse command-line arguments. + args, err := argsParse(nil) + if err != nil { + stderrLogger.Fatalf("failed to parse command line: %v", err) + } + + helper := apihelper.APIHelpers(apihelper.K8sHelpers{AnnotationNs: annotationNs, + LabelNs: labelNs}) + + if !args.noPublish { + err := updateMasterNode(helper) + if err != nil { + stderrLogger.Fatalf("failed to update master node: %v", err) + } + } + + // Create server listening for TCP connections + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", args.port)) + if err != nil { + stderrLogger.Fatalf("failed to listen: %v", err) + } + grpcServer := grpc.NewServer() + pb.RegisterLabelerServer(grpcServer, &labelerServer{args: args, apiHelper: helper}) + stdoutLogger.Printf("gRPC server serving on port: %d", args.port) + grpcServer.Serve(lis) +} + +// argsParse parses the command line arguments passed to the program. +// The argument argv is passed only for testing purposes. +func argsParse(argv []string) (Args, error) { + args := Args{} + usage := fmt.Sprintf(`%s. + + Usage: + %s [--no-publish] [--label-whitelist=] [--port=] + %s -h | --help + %s --version + + Options: + -h --help Show this screen. + --version Output version and exit. + --port= Port on which to listen for connections. + [Default: 8080] + --no-publish Do not publish feature labels + --label-whitelist= Regular expression to filter label names to + publish to the Kubernetes API server. [Default: ]`, + ProgramName, + ProgramName, + ProgramName, + ProgramName, + ) + + arguments, _ := docopt.Parse(usage, argv, true, + fmt.Sprintf("%s %s", ProgramName, version.Get()), false) + + // Parse argument values as usable types. + var err error + args.noPublish = arguments["--no-publish"].(bool) + args.port, err = strconv.Atoi(arguments["--port"].(string)) + if err != nil { + return args, fmt.Errorf("invalid --port defined: %s", err) + } + args.labelWhiteList, err = regexp.Compile(arguments["--label-whitelist"].(string)) + if err != nil { + return args, fmt.Errorf("error parsing whitelist regex (%s): %s", arguments["--label-whitelist"], err) + } + + return args, nil +} + +// Advertise NFD master information +func updateMasterNode(helper apihelper.APIHelpers) error { + cli, err := helper.GetClient() + if err != nil { + return err + } + node, err := helper.GetNode(cli, os.Getenv(NodeNameEnv)) + if err != nil { + return err + } + + // Advertise NFD version as an annotation + helper.AddAnnotations(node, Annotations{"master.version": version.Get()}) + err = helper.UpdateNode(cli, node) + if err != nil { + stderrLogger.Printf("can't update node: %s", err.Error()) + return err + } + + return nil +} + +// Filter labels if whitelist has been defined +func filterFeatureLabels(labels *Labels, labelWhiteList *regexp.Regexp) { + for name := range *labels { + // Skip if label doesn't match labelWhiteList + if !labelWhiteList.MatchString(name) { + stderrLogger.Printf("%s does not match the whitelist (%s) and will not be published.", name, labelWhiteList.String()) + delete(*labels, name) + } + } +} + +// Implement LabelerServer +type labelerServer struct { + args Args + apiHelper apihelper.APIHelpers +} + +// Service SetLabels +func (s *labelerServer) SetLabels(c context.Context, r *pb.SetLabelsRequest) (*pb.SetLabelsReply, error) { + stdoutLogger.Printf("REQUEST Node: %s NFD-version: %s Labels: %s", r.NodeName, r.NfdVersion, r.Labels) + if !s.args.noPublish { + // Advertise NFD worker version and label names as annotations + keys := make([]string, 0, len(r.Labels)) + for k, _ := range r.Labels { + keys = append(keys, k) + } + sort.Strings(keys) + annotations := Annotations{"worker.version": r.NfdVersion, + "feature-labels": strings.Join(keys, ",")} + + err := updateNodeFeatures(s.apiHelper, r.NodeName, r.Labels, annotations) + if err != nil { + stderrLogger.Printf("failed to advertise labels: %s", err.Error()) + return &pb.SetLabelsReply{}, err + } + } + return &pb.SetLabelsReply{}, nil +} + +// advertiseFeatureLabels advertises the feature labels to a Kubernetes node +// via the API server. +func updateNodeFeatures(helper apihelper.APIHelpers, nodeName string, labels Labels, annotations Annotations) error { + cli, err := helper.GetClient() + if err != nil { + return err + } + + // Get the worker node object + node, err := helper.GetNode(cli, nodeName) + if err != nil { + return err + } + + // Remove old labels + if l, ok := node.Annotations[annotationNs+"feature-labels"]; ok { + oldLabels := strings.Split(l, ",") + helper.RemoveLabels(node, oldLabels) + } + + // Also, remove all labels with the old prefix, and the old version label + helper.RemoveLabelsWithPrefix(node, "node.alpha.kubernetes-incubator.io/nfd") + helper.RemoveLabelsWithPrefix(node, "node.alpha.kubernetes-incubator.io/node-feature-discovery") + + // Add labels to the node object. + helper.AddLabels(node, labels) + + // Add annotations + helper.AddAnnotations(node, annotations) + + // Send the updated node to the apiserver. + err = helper.UpdateNode(cli, node) + if err != nil { + stderrLogger.Printf("can't update node: %s", err.Error()) + return err + } + + return nil +} diff --git a/main.go b/cmd/nfd-worker/main.go similarity index 63% rename from main.go rename to cmd/nfd-worker/main.go index a236c6283..1ccebdfcc 100644 --- a/main.go +++ b/cmd/nfd-worker/main.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( @@ -6,17 +22,15 @@ import ( "log" "os" "regexp" - "sort" "strings" "time" "github.com/docopt/docopt-go" "github.com/ghodss/yaml" - api "k8s.io/api/core/v1" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "golang.org/x/net/context" + "google.golang.org/grpc" "k8s.io/apimachinery/pkg/util/validation" - k8sclient "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" + pb "sigs.k8s.io/node-feature-discovery/pkg/labeler" "sigs.k8s.io/node-feature-discovery/pkg/version" "sigs.k8s.io/node-feature-discovery/source" "sigs.k8s.io/node-feature-discovery/source/cpu" @@ -36,14 +50,8 @@ import ( ) const ( - // ProgramName is the canonical name of this discovery program. - ProgramName = "node-feature-discovery" - - // Namespace is the prefix for all published labels. - labelNs = "feature.node.kubernetes.io/" - - // Namespace is the prefix for all published labels. - annotationNs = "nfd.node.kubernetes.io/" + // ProgramName is the canonical name of this program + ProgramName = "nfd-worker" // NodeNameEnv is the environment variable that contains this node's name. NodeNameEnv = "NODE_NAME" @@ -71,34 +79,6 @@ type Labels map[string]string // Annotations are used for NFD-related node metadata type Annotations map[string]string -// APIHelpers represents a set of API helpers for Kubernetes -type APIHelpers interface { - // GetClient returns a client - GetClient() (*k8sclient.Clientset, error) - - // GetNode returns the Kubernetes node on which this container is running. - GetNode(*k8sclient.Clientset) (*api.Node, error) - - // RemoveLabelsWithPrefix removes labels from the supplied node that contain the - // search string provided. In order to publish the changes, the node must - // subsequently be updated via the API server using the client library. - RemoveLabelsWithPrefix(*api.Node, string) - - // RemoveLabels removes NFD labels from a node object - RemoveLabels(*api.Node, []string) - - // AddLabels adds new NFD labels to the node object. - // In order to publish the labels, the node must be subsequently updated via the - // API server using the client library. - AddLabels(*api.Node, Labels) - - // Add annotations - AddAnnotations(*api.Node, Annotations) - - // UpdateNode updates the node via the API server using a client. - UpdateNode(*k8sclient.Clientset, *api.Node) error -} - // Command line arguments type Args struct { labelWhiteList string @@ -106,6 +86,7 @@ type Args struct { noPublish bool options string oneshot bool + server string sleepInterval time.Duration sources []string } @@ -115,13 +96,16 @@ func main() { if version.Get() == "undefined" { stderrLogger.Fatalf("version not set! Set -ldflags \"-X sigs.k8s.io/node-feature-discovery/pkg/version.version=`git describe --tags --dirty --always`\" during build or run.") } - stdoutLogger.Printf("Node Feature Discovery %s", version.Get()) + stdoutLogger.Printf("Node Feature Discovery Worker %s", version.Get()) // Parse command-line arguments. - args := argsParse(nil) + args, err := argsParse(nil) + if err != nil { + stderrLogger.Fatalf("failed to parse command line: %v", err) + } // Parse config - err := configParse(args.configFile, args.options) + err = configParse(args.configFile, args.options) if err != nil { stderrLogger.Print(err) } @@ -132,16 +116,25 @@ func main() { stderrLogger.Fatalf("error occurred while configuring parameters: %s", err.Error()) } - helper := APIHelpers(k8sHelpers{}) + // Connect to NFD server + opts := []grpc.DialOption{grpc.WithInsecure()} + conn, err := grpc.Dial(args.server, opts...) + if err != nil { + stderrLogger.Fatalf("failed to connect: %v", err) + } + defer conn.Close() + client := pb.NewLabelerClient(conn) for { // Get the set of feature labels. labels := createFeatureLabels(enabledSources, labelWhiteList) // Update the node with the feature labels. - err = updateNodeWithFeatureLabels(helper, args.noPublish, labels) - if err != nil { - stderrLogger.Fatalf("error occurred while updating node with feature labels: %s", err.Error()) + if !args.noPublish { + err := advertiseFeatureLabels(client, labels) + if err != nil { + stderrLogger.Fatalf("failed to advertise labels: %s", err.Error()) + } } if args.oneshot { @@ -151,6 +144,7 @@ func main() { if args.sleepInterval > 0 { time.Sleep(args.sleepInterval) } else { + conn.Close() // Sleep forever select {} } @@ -159,13 +153,14 @@ func main() { // argsParse parses the command line arguments passed to the program. // The argument argv is passed only for testing purposes. -func argsParse(argv []string) (args Args) { +func argsParse(argv []string) (Args, error) { + args := Args{} usage := fmt.Sprintf(`%s. Usage: %s [--no-publish] [--sources=] [--label-whitelist=] [--oneshot | --sleep-interval=] [--config=] - [--options=] + [--options=] [--server=] %s -h | --help %s --version @@ -173,12 +168,14 @@ func argsParse(argv []string) (args Args) { -h --help Show this screen. --version Output version and exit. --config= Config file to use. - [Default: /etc/kubernetes/node-feature-discovery/node-feature-discovery.conf] + [Default: /etc/kubernetes/node-feature-discovery/nfd-worker.conf] --options= Specify config options from command line. Config options are specified in the same format as in the config file (i.e. json or yaml). These options will override settings read from the config file. [Default: ] + --server= NFD server address to connecto to. + [Default: localhost:8080] --sources= Comma separated list of feature sources. [Default: cpu,cpuid,iommu,kernel,local,memory,network,pci,pstate,rdt,storage,system] --no-publish Do not publish discovered features to the @@ -203,6 +200,7 @@ func argsParse(argv []string) (args Args) { args.configFile = arguments["--config"].(string) args.noPublish = arguments["--no-publish"].(bool) args.options = arguments["--options"].(string) + args.server = arguments["--server"].(string) args.sources = strings.Split(arguments["--sources"].(string), ",") args.labelWhiteList = arguments["--label-whitelist"].(string) args.oneshot = arguments["--oneshot"].(bool) @@ -210,14 +208,14 @@ func argsParse(argv []string) (args Args) { // Check that sleep interval has a sane value if err != nil { - stderrLogger.Fatalf("invalid --sleep-interval specified: %s", err.Error()) + return args, fmt.Errorf("invalid --sleep-interval specified: %s", err.Error()) } if args.sleepInterval > 0 && args.sleepInterval < time.Second { stderrLogger.Printf("WARNING: too short sleep-intervall specified (%s), forcing to 1s", args.sleepInterval.String()) args.sleepInterval = time.Second } - return args + return args, nil } // Parse configuration options @@ -319,28 +317,6 @@ func createFeatureLabels(sources []source.FeatureSource, labelWhiteList *regexp. return labels } -// updateNodeWithFeatureLabels updates the node with the feature labels, unless -// disabled via --no-publish flag. -func updateNodeWithFeatureLabels(helper APIHelpers, noPublish bool, labels Labels) error { - if !noPublish { - // Advertise NFD version and label names as annotations - keys := make([]string, 0, len(labels)) - for k, _ := range labels { - keys = append(keys, k) - } - sort.Strings(keys) - annotations := Annotations{"version": version.Get(), - "feature-labels": strings.Join(keys, ",")} - - err := advertiseFeatureLabels(helper, labels, annotations) - if err != nil { - stderrLogger.Printf("failed to advertise labels: %s", err.Error()) - return err - } - } - return nil -} - // getFeatureLabels returns node labels for features discovered by the // supplied source. func getFeatureLabels(source source.FeatureSource) (labels Labels, err error) { @@ -388,114 +364,23 @@ func getFeatureLabels(source source.FeatureSource) (labels Labels, err error) { } // advertiseFeatureLabels advertises the feature labels to a Kubernetes node -// via the API server. -func advertiseFeatureLabels(helper APIHelpers, labels Labels, annotations Annotations) error { - cli, err := helper.GetClient() - if err != nil { - stderrLogger.Printf("can't get kubernetes client: %s", err.Error()) - return err - } +// via the NFD server. +func advertiseFeatureLabels(client pb.LabelerClient, labels Labels) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() - // Get the current node. - node, err := helper.GetNode(cli) - if err != nil { - stderrLogger.Printf("failed to get node: %s", err.Error()) - return err - } - - // Remove old labels - if l, ok := node.Annotations[annotationNs+"feature-labels"]; ok { - oldLabels := strings.Split(l, ",") - helper.RemoveLabels(node, oldLabels) - } - - // Also, remove all labels with the old prefix, and the old version label - helper.RemoveLabelsWithPrefix(node, "node.alpha.kubernetes-incubator.io/nfd") - helper.RemoveLabelsWithPrefix(node, "node.alpha.kubernetes-incubator.io/node-feature-discovery") - - // Add labels to the node object. - helper.AddLabels(node, labels) - - // Add annotations - helper.AddAnnotations(node, annotations) - - // Send the updated node to the apiserver. - err = helper.UpdateNode(cli, node) - if err != nil { - stderrLogger.Printf("can't update node: %s", err.Error()) - return err - } - - return nil -} - -// Implements main.APIHelpers -type k8sHelpers struct{} - -func (h k8sHelpers) GetClient() (*k8sclient.Clientset, error) { - // Set up an in-cluster K8S client. - config, err := restclient.InClusterConfig() - if err != nil { - return nil, err - } - clientset, err := k8sclient.NewForConfig(config) - if err != nil { - return nil, err - } - return clientset, nil -} - -func (h k8sHelpers) GetNode(cli *k8sclient.Clientset) (*api.Node, error) { - // Get the pod name and pod namespace from the env variables nodeName := os.Getenv(NodeNameEnv) stdoutLogger.Printf("%s: %s", NodeNameEnv, nodeName) - // Get the node object using node name - node, err := cli.Core().Nodes().Get(nodeName, meta_v1.GetOptions{}) - if err != nil { - stderrLogger.Printf("can't get node: %s", err.Error()) - return nil, err - } - - return node, nil -} - -// RemoveLabelsWithPrefix searches through all labels on Node n and removes -// any where the key contain the search string. -func (h k8sHelpers) RemoveLabelsWithPrefix(n *api.Node, search string) { - for k := range n.Labels { - if strings.Contains(k, search) { - delete(n.Labels, k) - } - } -} - -// RemoveLabels removes given NFD labels -func (h k8sHelpers) RemoveLabels(n *api.Node, labelNames []string) { - for _, l := range labelNames { - delete(n.Labels, labelNs+l) - } -} - -func (h k8sHelpers) AddLabels(n *api.Node, labels Labels) { - for k, v := range labels { - n.Labels[labelNs+k] = v - } -} - -// Add Annotations to the Node object -func (h k8sHelpers) AddAnnotations(n *api.Node, annotations Annotations) { - for k, v := range annotations { - n.Annotations[annotationNs+k] = v - } -} - -func (h k8sHelpers) UpdateNode(c *k8sclient.Clientset, n *api.Node) error { - // Send the updated node to the apiserver. - _, err := c.Core().Nodes().Update(n) + labelReq := pb.SetLabelsRequest{Labels: labels, + NfdVersion: version.Get(), + NodeName: nodeName} + rsp, err := client.SetLabels(ctx, &labelReq) if err != nil { + stderrLogger.Printf("failed to set node labels: %v", err) return err } + log.Printf("RESPONSE: %s", rsp) return nil } diff --git a/label-nodes.sh b/label-nodes.sh index 5ff95ae01..098f567b1 100755 --- a/label-nodes.sh +++ b/label-nodes.sh @@ -6,5 +6,5 @@ NumNodes=$(kubectl get nodes | grep -i ' ready ' | wc -l) # We request a specific hostPort in the job spec to limit the number of pods # that run on a node to one. As a result, one pod runs on each node in parallel # We set the NODE_NAME environemnt variable to get the Kubernetes node object. -sed -e "s/COMPLETION_COUNT/$NumNodes/" -e "s/PARALLELISM_COUNT/$NumNodes/" node-feature-discovery-job.yaml.template > node-feature-discovery-job.yaml +sed -e "s/COMPLETION_COUNT/$NumNodes/" -e "s/PARALLELISM_COUNT/$NumNodes/" nfd-worker-job.yaml.template > node-feature-discovery-job.yaml kubectl create -f node-feature-discovery-job.yaml diff --git a/nfd-master.yaml.template b/nfd-master.yaml.template new file mode 100644 index 000000000..86901c1ce --- /dev/null +++ b/nfd-master.yaml.template @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nfd-master + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nfd-master +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nfd-master +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nfd-master +subjects: +- kind: ServiceAccount + name: nfd-master + namespace: default +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: nfd-master + name: nfd-master +spec: + selector: + matchLabels: + app: nfd-master + template: + metadata: + labels: + app: nfd-master + spec: + serviceAccount: node-feature-discovery + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "" + effect: "NoSchedule" + containers: + - env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: quay.io/kubernetes_incubator/node-feature-discovery:v0.3.0 + name: nfd-master + command: + - "nfd-master" +--- +apiVersion: v1 +kind: Service +metadata: + name: nfd-master +spec: + selector: + app: nfd-master + ports: + - protocol: TCP + port: 8080 + type: ClusterIP diff --git a/node-feature-discovery-daemonset.yaml.template b/nfd-worker-daemonset.yaml.template similarity index 80% rename from node-feature-discovery-daemonset.yaml.template rename to nfd-worker-daemonset.yaml.template index 4411b95f0..09245234c 100644 --- a/node-feature-discovery-daemonset.yaml.template +++ b/nfd-worker-daemonset.yaml.template @@ -2,19 +2,18 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: - app: node-feature-discovery - name: node-feature-discovery + app: nfd-worker + name: nfd-worker spec: selector: matchLabels: - app: node-feature-discovery + app: nfd-worker template: metadata: labels: - app: node-feature-discovery + app: nfd-worker spec: hostNetwork: true - serviceAccount: node-feature-discovery containers: - env: - name: NODE_NAME @@ -22,9 +21,12 @@ spec: fieldRef: fieldPath: spec.nodeName image: quay.io/kubernetes_incubator/node-feature-discovery:v0.3.0 - name: node-feature-discovery + name: nfd-worker + command: + - "nfd-worker" args: - "--sleep-interval=60s" + - "--server=$(NFD_MASTER_SERVICE_HOST):$(NFD_MASTER_SERVICE_PORT)" volumeMounts: - name: host-boot mountPath: "/host-boot" diff --git a/node-feature-discovery-job.yaml.template b/nfd-worker-job.yaml.template similarity index 87% rename from node-feature-discovery-job.yaml.template rename to nfd-worker-job.yaml.template index 8954b27b9..fdbb778ff 100644 --- a/node-feature-discovery-job.yaml.template +++ b/nfd-worker-job.yaml.template @@ -3,7 +3,7 @@ kind: Job metadata: labels: app: node-feature-discovery - name: node-feature-discovery + name: nfd-worker spec: completions: COMPLETION_COUNT parallelism: PARALLELISM_COUNT @@ -13,7 +13,6 @@ spec: app: node-feature-discovery spec: hostNetwork: true - serviceAccount: node-feature-discovery containers: - env: - name: NODE_NAME @@ -21,9 +20,12 @@ spec: fieldRef: fieldPath: spec.nodeName image: quay.io/kubernetes_incubator/node-feature-discovery:v0.3.0 - name: node-feature-discovery + name: nfd-worker + command: + - "nfd-worker" args: - "--oneshot" + - "--server=$(NFD_MASTER_SERVICE_HOST):$(NFD_MASTER_SERVICE_PORT)" ports: - containerPort: 7156 hostPort: 7156 diff --git a/node-feature-discovery.conf.example b/nfd-worker.conf.example similarity index 100% rename from node-feature-discovery.conf.example rename to nfd-worker.conf.example diff --git a/pkg/apihelper/apihelpers.go b/pkg/apihelper/apihelpers.go new file mode 100644 index 000000000..62ecfe892 --- /dev/null +++ b/pkg/apihelper/apihelpers.go @@ -0,0 +1,50 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apihelper + +import ( + api "k8s.io/api/core/v1" + k8sclient "k8s.io/client-go/kubernetes" +) + +// APIHelpers represents a set of API helpers for Kubernetes +type APIHelpers interface { + // GetClient returns a client + GetClient() (*k8sclient.Clientset, error) + + // GetNode returns the Kubernetes node on which this container is running. + GetNode(*k8sclient.Clientset, string) (*api.Node, error) + + // RemoveLabelsWithPrefix removes labels from the supplied node that contain the + // search string provided. In order to publish the changes, the node must + // subsequently be updated via the API server using the client library. + RemoveLabelsWithPrefix(*api.Node, string) + + // RemoveLabels removes NFD labels from a node object + RemoveLabels(*api.Node, []string) + + // AddLabels adds new NFD labels to the node object. + // In order to publish the labels, the node must be subsequently updated via the + // API server using the client library. + AddLabels(*api.Node, map[string]string) + + // Add annotations + AddAnnotations(*api.Node, map[string]string) + + // UpdateNode updates the node via the API server using a client. + UpdateNode(*k8sclient.Clientset, *api.Node) error +} diff --git a/pkg/apihelper/k8shelpers.go b/pkg/apihelper/k8shelpers.go new file mode 100644 index 000000000..3747847d4 --- /dev/null +++ b/pkg/apihelper/k8shelpers.go @@ -0,0 +1,95 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apihelper + +import ( + "strings" + + api "k8s.io/api/core/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sclient "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" +) + +// Implements APIHelpers +type K8sHelpers struct { + AnnotationNs string + LabelNs string +} + +func (h K8sHelpers) GetClient() (*k8sclient.Clientset, error) { + // Set up an in-cluster K8S client. + config, err := restclient.InClusterConfig() + if err != nil { + return nil, err + } + clientset, err := k8sclient.NewForConfig(config) + if err != nil { + return nil, err + } + return clientset, nil +} + +func (h K8sHelpers) GetNode(cli *k8sclient.Clientset, nodeName string) (*api.Node, error) { + // Get the node object using node name + node, err := cli.Core().Nodes().Get(nodeName, meta_v1.GetOptions{}) + if err != nil { + return nil, err + } + + return node, nil +} + +// RemoveLabelsWithPrefix searches through all labels on Node n and removes +// any where the key contain the search string. +func (h K8sHelpers) RemoveLabelsWithPrefix(n *api.Node, search string) { + for k := range n.Labels { + if strings.Contains(k, search) { + delete(n.Labels, k) + } + } +} + +// RemoveLabels removes given NFD labels +func (h K8sHelpers) RemoveLabels(n *api.Node, labelNames []string) { + for _, l := range labelNames { + delete(n.Labels, h.LabelNs+l) + } +} + +func (h K8sHelpers) AddLabels(n *api.Node, labels map[string]string) { + for k, v := range labels { + n.Labels[h.LabelNs+k] = v + } +} + +// Add Annotations to the Node object +func (h K8sHelpers) AddAnnotations(n *api.Node, annotations map[string]string) { + for k, v := range annotations { + n.Annotations[h.AnnotationNs+k] = v + } +} + +func (h K8sHelpers) UpdateNode(c *k8sclient.Clientset, n *api.Node) error { + // Send the updated node to the apiserver. + _, err := c.Core().Nodes().Update(n) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/labeler/labeler.pb.go b/pkg/labeler/labeler.pb.go new file mode 100644 index 000000000..05bd259d3 --- /dev/null +++ b/pkg/labeler/labeler.pb.go @@ -0,0 +1,206 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: labeler.proto + +package labeler + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type SetLabelsRequest struct { + NfdVersion string `protobuf:"bytes,1,opt,name=nfd_version,json=nfdVersion" json:"nfd_version,omitempty"` + NodeName string `protobuf:"bytes,2,opt,name=node_name,json=nodeName" json:"node_name,omitempty"` + Labels map[string]string `protobuf:"bytes,3,rep,name=labels" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SetLabelsRequest) Reset() { *m = SetLabelsRequest{} } +func (m *SetLabelsRequest) String() string { return proto.CompactTextString(m) } +func (*SetLabelsRequest) ProtoMessage() {} +func (*SetLabelsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_labeler_06d914ce56652184, []int{0} +} +func (m *SetLabelsRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SetLabelsRequest.Unmarshal(m, b) +} +func (m *SetLabelsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SetLabelsRequest.Marshal(b, m, deterministic) +} +func (dst *SetLabelsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_SetLabelsRequest.Merge(dst, src) +} +func (m *SetLabelsRequest) XXX_Size() int { + return xxx_messageInfo_SetLabelsRequest.Size(m) +} +func (m *SetLabelsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_SetLabelsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_SetLabelsRequest proto.InternalMessageInfo + +func (m *SetLabelsRequest) GetNfdVersion() string { + if m != nil { + return m.NfdVersion + } + return "" +} + +func (m *SetLabelsRequest) GetNodeName() string { + if m != nil { + return m.NodeName + } + return "" +} + +func (m *SetLabelsRequest) GetLabels() map[string]string { + if m != nil { + return m.Labels + } + return nil +} + +type SetLabelsReply struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SetLabelsReply) Reset() { *m = SetLabelsReply{} } +func (m *SetLabelsReply) String() string { return proto.CompactTextString(m) } +func (*SetLabelsReply) ProtoMessage() {} +func (*SetLabelsReply) Descriptor() ([]byte, []int) { + return fileDescriptor_labeler_06d914ce56652184, []int{1} +} +func (m *SetLabelsReply) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SetLabelsReply.Unmarshal(m, b) +} +func (m *SetLabelsReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SetLabelsReply.Marshal(b, m, deterministic) +} +func (dst *SetLabelsReply) XXX_Merge(src proto.Message) { + xxx_messageInfo_SetLabelsReply.Merge(dst, src) +} +func (m *SetLabelsReply) XXX_Size() int { + return xxx_messageInfo_SetLabelsReply.Size(m) +} +func (m *SetLabelsReply) XXX_DiscardUnknown() { + xxx_messageInfo_SetLabelsReply.DiscardUnknown(m) +} + +var xxx_messageInfo_SetLabelsReply proto.InternalMessageInfo + +func init() { + proto.RegisterType((*SetLabelsRequest)(nil), "labeler.SetLabelsRequest") + proto.RegisterMapType((map[string]string)(nil), "labeler.SetLabelsRequest.LabelsEntry") + proto.RegisterType((*SetLabelsReply)(nil), "labeler.SetLabelsReply") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for Labeler service + +type LabelerClient interface { + SetLabels(ctx context.Context, in *SetLabelsRequest, opts ...grpc.CallOption) (*SetLabelsReply, error) +} + +type labelerClient struct { + cc *grpc.ClientConn +} + +func NewLabelerClient(cc *grpc.ClientConn) LabelerClient { + return &labelerClient{cc} +} + +func (c *labelerClient) SetLabels(ctx context.Context, in *SetLabelsRequest, opts ...grpc.CallOption) (*SetLabelsReply, error) { + out := new(SetLabelsReply) + err := grpc.Invoke(ctx, "/labeler.Labeler/SetLabels", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for Labeler service + +type LabelerServer interface { + SetLabels(context.Context, *SetLabelsRequest) (*SetLabelsReply, error) +} + +func RegisterLabelerServer(s *grpc.Server, srv LabelerServer) { + s.RegisterService(&_Labeler_serviceDesc, srv) +} + +func _Labeler_SetLabels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetLabelsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(LabelerServer).SetLabels(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/labeler.Labeler/SetLabels", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(LabelerServer).SetLabels(ctx, req.(*SetLabelsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _Labeler_serviceDesc = grpc.ServiceDesc{ + ServiceName: "labeler.Labeler", + HandlerType: (*LabelerServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "SetLabels", + Handler: _Labeler_SetLabels_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "labeler.proto", +} + +func init() { proto.RegisterFile("labeler.proto", fileDescriptor_labeler_06d914ce56652184) } + +var fileDescriptor_labeler_06d914ce56652184 = []byte{ + // 220 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0x49, 0x4c, 0x4a, + 0xcd, 0x49, 0x2d, 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x87, 0x72, 0x95, 0x4e, 0x31, + 0x72, 0x09, 0x04, 0xa7, 0x96, 0xf8, 0x80, 0xb8, 0xc5, 0x41, 0xa9, 0x85, 0xa5, 0xa9, 0xc5, 0x25, + 0x42, 0xf2, 0x5c, 0xdc, 0x79, 0x69, 0x29, 0xf1, 0x65, 0xa9, 0x45, 0xc5, 0x99, 0xf9, 0x79, 0x12, + 0x8c, 0x0a, 0x8c, 0x1a, 0x9c, 0x41, 0x5c, 0x79, 0x69, 0x29, 0x61, 0x10, 0x11, 0x21, 0x69, 0x2e, + 0xce, 0xbc, 0xfc, 0x94, 0xd4, 0xf8, 0xbc, 0xc4, 0xdc, 0x54, 0x09, 0x26, 0xb0, 0x34, 0x07, 0x48, + 0xc0, 0x2f, 0x31, 0x37, 0x55, 0xc8, 0x96, 0x8b, 0x0d, 0x6c, 0x7a, 0xb1, 0x04, 0xb3, 0x02, 0xb3, + 0x06, 0xb7, 0x91, 0xaa, 0x1e, 0xcc, 0x6e, 0x74, 0x8b, 0xf4, 0x20, 0x3c, 0xd7, 0xbc, 0x92, 0xa2, + 0xca, 0x20, 0xa8, 0x26, 0x29, 0x4b, 0x2e, 0x6e, 0x24, 0x61, 0x21, 0x01, 0x2e, 0xe6, 0xec, 0xd4, + 0x4a, 0xa8, 0x1b, 0x40, 0x4c, 0x21, 0x11, 0x2e, 0xd6, 0xb2, 0xc4, 0x9c, 0x52, 0x98, 0xc5, 0x10, + 0x8e, 0x15, 0x93, 0x05, 0xa3, 0x92, 0x00, 0x17, 0x1f, 0x92, 0x15, 0x05, 0x39, 0x95, 0x46, 0x3e, + 0x5c, 0xec, 0x3e, 0x10, 0xcb, 0x85, 0x1c, 0xb9, 0x38, 0xe1, 0x92, 0x42, 0x92, 0x38, 0xdd, 0x24, + 0x25, 0x8e, 0x4d, 0xaa, 0x20, 0xa7, 0x52, 0x89, 0x21, 0x89, 0x0d, 0x1c, 0x78, 0xc6, 0x80, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x2f, 0x68, 0x0f, 0x06, 0x4d, 0x01, 0x00, 0x00, +} diff --git a/pkg/labeler/labeler.proto b/pkg/labeler/labeler.proto new file mode 100644 index 000000000..13811e36a --- /dev/null +++ b/pkg/labeler/labeler.proto @@ -0,0 +1,35 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +syntax = "proto3"; + +//option go_package = "labeler"; + +package labeler; + +service Labeler{ + rpc SetLabels(SetLabelsRequest) returns (SetLabelsReply) {} +} + +message SetLabelsRequest { + string nfd_version = 1; + string node_name = 2; + map labels = 3; +} + +message SetLabelsReply { +} + diff --git a/rbac.yaml b/rbac.yaml deleted file mode 100644 index a726131ad..000000000 --- a/rbac.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: node-feature-discovery ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: node-feature-discovery -rules: -- apiGroups: - - "" - resources: - - pods - - nodes - verbs: - - get - - patch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: node-feature-discovery -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: node-feature-discovery -subjects: -- kind: ServiceAccount - name: node-feature-discovery - namespace: default - -