Merge pull request #563 from ArangoGutierrez/devel/issue522

Add Readiness and liveliness probes to nfd-master

Dockerfile

@@ -4,6 +4,11 @@ ARG BASE_IMAGE_MINIMAL
 # Build node feature discovery
 FROM golang:1.16.7-buster as builder
 
+# Download the grpc_health_probe bin
+RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
+    wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
+    chmod +x /bin/grpc_health_probe
+
 # Get (cache) deps in a separate layer
 COPY go.mod go.sum /go/node-feature-discovery/
 
@@ -21,7 +26,6 @@ RUN make install VERSION=$VERSION HOSTMOUNT_PREFIX=$HOSTMOUNT_PREFIX
 
 RUN make test
 
-
 # Create full variant of the production image
 FROM ${BASE_IMAGE_FULL} as full
 
@@ -33,6 +37,7 @@ ENV GRPC_GO_LOG_SEVERITY_LEVEL="INFO"
 
 COPY --from=builder /go/node-feature-discovery/deployment/components/worker-config/nfd-worker.conf.example /etc/kubernetes/node-feature-discovery/nfd-worker.conf
 COPY --from=builder /go/bin/* /usr/bin/
+COPY --from=builder /bin/grpc_health_probe /usr/bin/grpc_health_probe
 
 # Create minimal variant of the production image
 FROM ${BASE_IMAGE_MINIMAL} as minimal
@@ -45,3 +50,4 @@ ENV GRPC_GO_LOG_SEVERITY_LEVEL="INFO"
 
 COPY --from=builder /go/node-feature-discovery/deployment/components/worker-config/nfd-worker.conf.example /etc/kubernetes/node-feature-discovery/nfd-worker.conf
 COPY --from=builder /go/bin/* /usr/bin/
+COPY --from=builder /bin/grpc_health_probe /usr/bin/grpc_health_probe

deployment/base/master-worker-combined/master-worker-daemonset.yaml

@@ -19,6 +19,17 @@ spec:
       - name: nfd-master
         image: gcr.io/k8s-staging-nfd/node-feature-discovery:master
         imagePullPolicy: Always
+          livenessProbe:
+            exec:
+              command: ["/usr/bin/grpc_health_probe", "-addr=:8080"]
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          readinessProbe:
+            exec:
+              command: ["/usr/bin/grpc_health_probe", "-addr=:8080"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            failureThreshold: 10
         command:
         - "nfd-master"
       - name: nfd-worker

deployment/base/master/master-deployment.yaml

@@ -20,6 +20,17 @@ spec:
         - name: nfd-master
           image: gcr.io/k8s-staging-nfd/node-feature-discovery:master
           imagePullPolicy: Always
+          livenessProbe:
+            exec:
+              command: ["/usr/bin/grpc_health_probe", "-addr=:8080"]
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          readinessProbe:
+            exec:
+              command: ["/usr/bin/grpc_health_probe", "-addr=:8080"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            failureThreshold: 10
           command:
             - "nfd-master"
           args: []

deployment/helm/node-feature-discovery/templates/master.yaml

@@ -32,6 +32,17 @@ spec:
             {{- toYaml .Values.master.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          livenessProbe:
+            exec:
+              command: ["/usr/bin/grpc_health_probe", "-addr=:8080"]
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          readinessProbe:
+            exec:
+              command: ["/usr/bin/grpc_health_probe", "-addr=:8080"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            failureThreshold: 10
           ports:
           - containerPort: 8080
             name: grpc

pkg/nfd-master/nfd-master.go

@@ -36,6 +36,8 @@ import (
 	api "k8s.io/api/core/v1"
 	"k8s.io/klog/v2"
 
+	"google.golang.org/grpc/health"
+	"google.golang.org/grpc/health/grpc_health_v1"
 	"sigs.k8s.io/node-feature-discovery/pkg/apihelper"
 	pb "sigs.k8s.io/node-feature-discovery/pkg/labeler"
 	"sigs.k8s.io/node-feature-discovery/pkg/utils"
@@ -186,6 +188,7 @@ func (m *nfdMaster) Run() error {
 	}
 	m.server = grpc.NewServer(serverOpts...)
 	pb.RegisterLabelerServer(m.server, m)
+	grpc_health_v1.RegisterHealthServer(m.server, health.NewServer())
 	klog.Infof("gRPC server serving on port: %d", m.args.Port)
 
 	// Run gRPC server