Merge branch 'master' into tests/multi-deployment

docs/Manual/Deployment/Kubernetes/Metrics.md

@@ -1,3 +1,10 @@
 # Metrics
 
-TBD
+The ArangoDB Kubernetes Operator (`kube-arangodb`) exposes metrics of
+its operations in a format that is compatible with [Prometheus](https://prometheus.io).
+
+The metrics are exposed through HTTPS on port `8528` under path `/metrics`.
+
+Look at [examples/metrics](https://github.com/arangodb/kube-arangodb/tree/master/examples/metrics)
+for examples of `Services` and `ServiceMonitors` you can use to integrate
+with Prometheus through the [Prometheus-Operator by CoreOS](https://github.com/coreos/prometheus-operator).

examples/metrics/deployment-operator-servicemonitor.yaml

@@ -0,0 +1,34 @@
+# This example shows how to integrate with the Prometheus Operator
+# to bring metrics from kube-arangodb to Prometheus.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: arango-deployment-operator
+  labels:
+    app: arango-deployment-operator
+spec:
+  selector:
+    app: arango-deployment-operator
+  ports:
+  - name: metrics
+    port: 8528
+
+---
+
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: arango-deployment-operator
+  labels:
+    team: frontend
+spec:
+  selector:
+    matchLabels:
+      app: arango-deployment-operator
+  endpoints:
+  - port: metrics
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
+

main.go

@@ -46,6 +46,7 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/client"
 	"github.com/arangodb/kube-arangodb/pkg/logging"
 	"github.com/arangodb/kube-arangodb/pkg/operator"
+	"github.com/arangodb/kube-arangodb/pkg/server"
 	"github.com/arangodb/kube-arangodb/pkg/util/constants"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	"github.com/arangodb/kube-arangodb/pkg/util/probe"
@@ -69,12 +70,13 @@ var (
 		Run: cmdMainRun,
 	}
 
-	logLevel   string
-	cliLog     = logging.NewRootLogger()
-	logService logging.Service
-	server     struct {
-		host string
-		port int
+	logLevel      string
+	cliLog        = logging.NewRootLogger()
+	logService    logging.Service
+	serverOptions struct {
+		host          string
+		port          int
+		tlsSecretName string
 	}
 	operatorOptions struct {
 		enableDeployment bool // Run deployment operator
@@ -89,8 +91,9 @@ var (
 
 func init() {
 	f := cmdMain.Flags()
-	f.StringVar(&server.host, "server.host", defaultServerHost, "Host to listen on")
-	f.IntVar(&server.port, "server.port", defaultServerPort, "Port to listen on")
+	f.StringVar(&serverOptions.host, "server.host", defaultServerHost, "Host to listen on")
+	f.IntVar(&serverOptions.port, "server.port", defaultServerPort, "Port to listen on")
+	f.StringVar(&serverOptions.tlsSecretName, "server.tls-secret-name", "", "Name of secret containing tls.crt & tls.key for HTTPS server (if empty, self-signed certificate is used)")
 	f.StringVar(&logLevel, "log.level", defaultLogLevel, "Set initial log level")
 	f.BoolVar(&operatorOptions.enableDeployment, "operator.deployment", false, "Enable to run the ArangoDeployment operator")
 	f.BoolVar(&operatorOptions.enableStorage, "operator.storage", false, "Enable to run the ArangoLocalStorage operator")
@@ -133,6 +136,10 @@ func cmdMainRun(cmd *cobra.Command, args []string) {
 	if len(name) == 0 {
 		cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodName)
 	}
+	ip := os.Getenv(constants.EnvOperatorPodIP)
+	if len(ip) == 0 {
+		cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodIP)
+	}
 
 	// Get host name
 	id, err := os.Hostname()
@@ -140,12 +147,29 @@ func cmdMainRun(cmd *cobra.Command, args []string) {
 		cliLog.Fatal().Err(err).Msg("Failed to get hostname")
 	}
 
-	http.HandleFunc("/health", probe.LivenessHandler)
-	http.HandleFunc("/ready/deployment", deploymentProbe.ReadyHandler)
-	http.HandleFunc("/ready/storage", storageProbe.ReadyHandler)
-	http.Handle("/metrics", prometheus.Handler())
-	listenAddr := net.JoinHostPort(server.host, strconv.Itoa(server.port))
-	go http.ListenAndServe(listenAddr, nil)
+	// Create kubernetes client
+	kubecli, err := k8sutil.NewKubeClient()
+	if err != nil {
+		cliLog.Fatal().Err(err).Msg("Failed to create Kubernetes client")
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/health", probe.LivenessHandler)
+	mux.HandleFunc("/ready/deployment", deploymentProbe.ReadyHandler)
+	mux.HandleFunc("/ready/storage", storageProbe.ReadyHandler)
+	mux.Handle("/metrics", prometheus.Handler())
+	listenAddr := net.JoinHostPort(serverOptions.host, strconv.Itoa(serverOptions.port))
+	if svr, err := server.NewServer(kubecli.CoreV1(), mux, server.Config{
+		Address:            listenAddr,
+		TLSSecretName:      serverOptions.tlsSecretName,
+		TLSSecretNamespace: namespace,
+		PodName:            name,
+		PodIP:              ip,
+	}); err != nil {
+		cliLog.Fatal().Err(err).Msg("Failed to create HTTP server")
+	} else {
+		go svr.Run()
+	}
 
 	cfg, deps, err := newOperatorConfigAndDeps(id+"-"+name, namespace, name)
 	if err != nil {

manifests/templates/deployment/deployment.yaml

@@ -12,6 +12,7 @@ spec:
     metadata:
       labels:
         name: {{ .Deployment.OperatorDeploymentName }}
+        app: arango-deployment-operator
     spec:
       containers:
       - name: operator
@@ -29,6 +30,10 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
+        - name: MY_POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
         ports:
         - name: metrics
           containerPort: 8528
@@ -36,11 +41,13 @@ spec:
           httpGet:
             path: /health
             port: 8528
+            scheme: HTTPS
           initialDelaySeconds: 5
           periodSeconds: 10
         readinessProbe:
           httpGet:
             path: /ready/deployment
             port: 8528
+            scheme: HTTPS
           initialDelaySeconds: 5
           periodSeconds: 10

manifests/templates/storage/deployment.yaml

@@ -20,6 +20,7 @@ spec:
     metadata:
       labels:
         name: {{ .Storage.OperatorDeploymentName }}
+        app: arango-storage-operator
     spec:
       serviceAccountName: {{ .Storage.Operator.ServiceAccountName }}
       containers:
@@ -37,6 +38,10 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
+        - name: MY_POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
         ports:
         - name: metrics
           containerPort: 8528
@@ -44,12 +49,14 @@ spec:
           httpGet:
             path: /health
             port: 8528
+            scheme: HTTPS
           initialDelaySeconds: 5
           periodSeconds: 10
         readinessProbe:
           httpGet:
             path: /ready/storage
             port: 8528
+            scheme: HTTPS
           initialDelaySeconds: 5
           periodSeconds: 10
       

pkg/deployment/deployment.go

@@ -137,7 +137,7 @@ func New(config Config, deps Dependencies, apiObject *api.ArangoDeployment) (*De
 	}
 	if config.AllowChaos {
 		d.chaosMonkey = chaos.NewMonkey(deps.Log, d)
-		d.chaosMonkey.Run(d.stopCh)
+		go d.chaosMonkey.Run(d.stopCh)
 	}
 
 	return d, nil

pkg/server/errors.go

@@ -0,0 +1,29 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package server
+
+import "github.com/pkg/errors"
+
+var (
+	maskAny = errors.WithStack
+)

pkg/server/server.go

@@ -0,0 +1,126 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package server
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"time"
+
+	certificates "github.com/arangodb-helper/go-certificates"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
+)
+
+// Config settings for the Server
+type Config struct {
+	Address            string // Address to listen on
+	TLSSecretName      string // Name of secret containing TLS certificate
+	TLSSecretNamespace string // Namespace of secret containing TLS certificate
+	PodName            string // Name of the Pod we're running in
+	PodIP              string // IP address of the Pod we're running in
+}
+
+// Server is the HTTPS server for the operator.
+type Server struct {
+	httpServer *http.Server
+}
+
+// NewServer creates a new server, fetching/preparing a TLS certificate.
+func NewServer(cli corev1.CoreV1Interface, handler http.Handler, cfg Config) (*Server, error) {
+	httpServer := &http.Server{
+		Addr:              cfg.Address,
+		Handler:           handler,
+		ReadTimeout:       time.Second * 30,
+		ReadHeaderTimeout: time.Second * 15,
+		WriteTimeout:      time.Second * 30,
+		TLSNextProto:      make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
+	}
+
+	var cert, key string
+	if cfg.TLSSecretName != "" && cfg.TLSSecretNamespace != "" {
+		// Load TLS certificate from secret
+		s, err := cli.Secrets(cfg.TLSSecretNamespace).Get(cfg.TLSSecretName, metav1.GetOptions{})
+		if err != nil {
+			return nil, maskAny(err)
+		}
+		certBytes, found := s.Data[v1.TLSCertKey]
+		if !found {
+			return nil, maskAny(fmt.Errorf("No %s found in secret %s", v1.TLSCertKey, cfg.TLSSecretName))
+		}
+		keyBytes, found := s.Data[v1.TLSPrivateKeyKey]
+		if !found {
+			return nil, maskAny(fmt.Errorf("No %s found in secret %s", v1.TLSPrivateKeyKey, cfg.TLSSecretName))
+		}
+		cert = string(certBytes)
+		key = string(keyBytes)
+	} else {
+		// Secret not specified, create our own TLS certificate
+		options := certificates.CreateCertificateOptions{
+			CommonName: cfg.PodName,
+			Hosts:      []string{cfg.PodName, cfg.PodIP},
+			ValidFrom:  time.Now(),
+			ValidFor:   time.Hour * 24 * 365 * 10,
+			IsCA:       false,
+			ECDSACurve: "P256",
+		}
+		var err error
+		cert, key, err = certificates.CreateCertificate(options, nil)
+		if err != nil {
+			return nil, maskAny(err)
+		}
+	}
+	tlsConfig, err := createTLSConfig(cert, key)
+	if err != nil {
+		return nil, maskAny(err)
+	}
+	tlsConfig.BuildNameToCertificate()
+	httpServer.TLSConfig = tlsConfig
+
+	return &Server{
+		httpServer: httpServer,
+	}, nil
+}
+
+// Run the server until the program stops.
+func (s *Server) Run() error {
+	if err := s.httpServer.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed {
+		return maskAny(err)
+	}
+	return nil
+}
+
+// createTLSConfig creates a TLS config based on given config
+func createTLSConfig(cert, key string) (*tls.Config, error) {
+	var result *tls.Config
+	c, err := tls.X509KeyPair([]byte(cert), []byte(key))
+	if err != nil {
+		return nil, maskAny(err)
+	}
+	result = &tls.Config{
+		Certificates: []tls.Certificate{c},
+	}
+	return result, nil
+}

pkg/util/constants/constants.go

@@ -26,6 +26,7 @@ const (
 	EnvOperatorNodeName     = "MY_NODE_NAME"
 	EnvOperatorPodName      = "MY_POD_NAME"
 	EnvOperatorPodNamespace = "MY_POD_NAMESPACE"
+	EnvOperatorPodIP        = "MY_POD_IP"
 
 	EnvArangodJWTSecret    = "ARANGOD_JWT_SECRET"    // Contains JWT secret for the ArangoDB cluster
 	EnvArangoSyncJWTSecret = "ARANGOSYNC_JWT_SECRET" // Contains JWT secret for the ArangoSync masters