nfd-master: support certificate rotation

Add a helper/wrapper in pkg/utils to handle gRPC server-side certificate
rotation.

pkg/nfd-master/nfd-master.go

@@ -18,9 +18,7 @@ package nfdmaster
 
 import (
 	"crypto/tls"
-	"crypto/x509"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"os"
 	"path"
@@ -93,6 +91,7 @@ type nfdMaster struct {
 	nodeName     string
 	annotationNs string
 	server       *grpc.Server
+	stop         chan struct{}
 	ready        chan bool
 	apihelper    apihelper.APIHelpers
 }
@@ -102,6 +101,7 @@ func NewNfdMaster(args *Args) (NfdMaster, error) {
 	nfd := &nfdMaster{args: *args,
 		nodeName: os.Getenv("NODE_NAME"),
 		ready:    make(chan bool, 1),
+		stop:     make(chan struct{}, 1),
 	}
 
 	if args.Instance == "" {
@@ -164,40 +164,61 @@ func (m *nfdMaster) Run() error {
 	close(m.ready)
 
 	serverOpts := []grpc.ServerOption{}
+	tlsConfig := utils.TlsConfig{}
+	// Create watcher for TLS cert files
+	certWatch, err := utils.CreateFsWatcher(time.Second, m.args.CertFile, m.args.KeyFile, m.args.CaFile)
+	if err != nil {
+		return err
+	}
 	// Enable mutual TLS authentication if --cert-file, --key-file or --ca-file
 	// is defined
 	if m.args.CertFile != "" || m.args.KeyFile != "" || m.args.CaFile != "" {
-		// Load cert for authenticating this server
+		if err := tlsConfig.UpdateConfig(m.args.CertFile, m.args.KeyFile, m.args.CaFile); err != nil {
-		cert, err := tls.LoadX509KeyPair(m.args.CertFile, m.args.KeyFile)
+			return err
-		if err != nil {
-			return fmt.Errorf("failed to load server certificate: %v", err)
-		}
-		// Load CA cert for client cert verification
-		caCert, err := ioutil.ReadFile(m.args.CaFile)
-		if err != nil {
-			return fmt.Errorf("failed to read root certificate file: %v", err)
-		}
-		caPool := x509.NewCertPool()
-		if ok := caPool.AppendCertsFromPEM(caCert); !ok {
-			return fmt.Errorf("failed to add certificate from %q", m.args.CaFile)
-		}
-		// Create TLS config
-		tlsConfig := &tls.Config{
-			Certificates: []tls.Certificate{cert},
-			ClientCAs:    caPool,
-			ClientAuth:   tls.RequireAndVerifyClientCert,
 		}
+
+		tlsConfig := &tls.Config{GetConfigForClient: tlsConfig.GetConfig}
 		serverOpts = append(serverOpts, grpc.Creds(credentials.NewTLS(tlsConfig)))
 	}
 	m.server = grpc.NewServer(serverOpts...)
 	pb.RegisterLabelerServer(m.server, m)
 	klog.Infof("gRPC server serving on port: %d", m.args.Port)
-	return m.server.Serve(lis)
+
+	// Run gRPC server
+	grpcErr := make(chan error, 1)
+	go func() {
+		defer lis.Close()
+		grpcErr <- m.server.Serve(lis)
+	}()
+
+	// NFD-Master main event loop
+	for {
+		select {
+		case <-certWatch.Events:
+			klog.Infof("reloading TLS certificates")
+			if err := tlsConfig.UpdateConfig(m.args.CertFile, m.args.KeyFile, m.args.CaFile); err != nil {
+				return err
+			}
+
+		case <-grpcErr:
+			return fmt.Errorf("gRPC server exited with an error: %v", err)
+
+		case <-m.stop:
+			klog.Infof("shutting down nfd-master")
+			certWatch.Close()
+			return nil
+		}
+	}
 }
 
 // Stop NfdMaster
 func (m *nfdMaster) Stop() {
 	m.server.Stop()
+
+	select {
+	case m.stop <- struct{}{}:
+	default:
+	}
 }
 
 // Wait until NfdMaster is able able to accept connections.

pkg/utils/tls.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"sync"
+)
+
+// TlsConfig is a TLS config wrapper/helper for cert rotation
+type TlsConfig struct {
+	sync.Mutex
+	config *tls.Config
+}
+
+// GetConfig returns the current TLS configuration. Intended to be used as the
+// GetConfigForClient callback in tls.Config.
+func (c *TlsConfig) GetConfig(*tls.ClientHelloInfo) (*tls.Config, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.config, nil
+}
+
+// UpdateConfig updates the wrapped TLS config
+func (c *TlsConfig) UpdateConfig(certFile, keyFile, caFile string) error {
+	c.Lock()
+	defer c.Unlock()
+
+	// Load cert for authenticating this server
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return fmt.Errorf("failed to load server certificate: %v", err)
+	}
+	// Load CA cert for client cert verification
+	caCert, err := ioutil.ReadFile(caFile)
+	if err != nil {
+		return fmt.Errorf("failed to read root certificate file: %v", err)
+	}
+	caPool := x509.NewCertPool()
+	if ok := caPool.AppendCertsFromPEM(caCert); !ok {
+		return fmt.Errorf("failed to add certificate from '%s'", caFile)
+	}
+
+	// Create TLS config
+	c.config = &tls.Config{
+		Certificates:       []tls.Certificate{cert},
+		ClientCAs:          caPool,
+		ClientAuth:         tls.RequireAndVerifyClientCert,
+		GetConfigForClient: c.GetConfig,
+	}
+	return nil
+}