prometheus: Load basic-auth secrets from ServiceMonitor namespace (#1619)

Basic-Auth Secrets for ServiceMonitor used to be loaded from the
prometheus'namespace. With this change, Prometheus will load the secret
from the servicemonitor namespace.

The commit also includes a basic-auth image for the integration test

pkg/prometheus/operator.go

@@ -955,12 +955,26 @@ func loadBasicAuthSecret(basicAuth *monitoringv1.BasicAuth, s *v1.SecretList) (B
 
 }
 
-func (c *Operator) loadBasicAuthSecrets(mons map[string]*monitoringv1.ServiceMonitor, remoteReads []monitoringv1.RemoteReadSpec, remoteWrites []monitoringv1.RemoteWriteSpec, s *v1.SecretList) (map[string]BasicAuthCredentials, error) {
+func (c *Operator) loadBasicAuthSecrets(mons map[string]*monitoringv1.ServiceMonitor, remoteReads []monitoringv1.RemoteReadSpec, remoteWrites []monitoringv1.RemoteWriteSpec, SecretsInPromNS *v1.SecretList) (map[string]BasicAuthCredentials, error) {
+	sMonSecretMap := make(map[string]*v1.SecretList)
+	for _, mon := range mons {
+		smNamespace := mon.Namespace
+		if sMonSecretMap[smNamespace] == nil {
+			msClient := c.kclient.CoreV1().Secrets(smNamespace)
+			listSecrets, err := msClient.List(metav1.ListOptions{})
+
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to retrieve secrets in namespace '%v' for servicemonitor '%v'", smNamespace, mon.Name)
+			}
+			sMonSecretMap[smNamespace] = listSecrets
+		}
+	}
+
 	secrets := map[string]BasicAuthCredentials{}
 	for _, mon := range mons {
 		for i, ep := range mon.Spec.Endpoints {
 			if ep.BasicAuth != nil {
-				credentials, err := loadBasicAuthSecret(ep.BasicAuth, s)
+				credentials, err := loadBasicAuthSecret(ep.BasicAuth, sMonSecretMap[mon.Namespace])
 				if err != nil {
 					return nil, fmt.Errorf("could not generate basicAuth for servicemonitor %s. %s", mon.Name, err)
 				}
@@ -971,7 +985,7 @@ func (c *Operator) loadBasicAuthSecrets(mons map[string]*monitoringv1.ServiceMon
 
 	for i, remote := range remoteReads {
 		if remote.BasicAuth != nil {
-			credentials, err := loadBasicAuthSecret(remote.BasicAuth, s)
+			credentials, err := loadBasicAuthSecret(remote.BasicAuth, SecretsInPromNS)
 			if err != nil {
 				return nil, fmt.Errorf("could not generate basicAuth for remote_read config %d. %s", i, err)
 			}
@@ -981,7 +995,7 @@ func (c *Operator) loadBasicAuthSecrets(mons map[string]*monitoringv1.ServiceMon
 
 	for i, remote := range remoteWrites {
 		if remote.BasicAuth != nil {
-			credentials, err := loadBasicAuthSecret(remote.BasicAuth, s)
+			credentials, err := loadBasicAuthSecret(remote.BasicAuth, SecretsInPromNS)
 			if err != nil {
 				return nil, fmt.Errorf("could not generate basicAuth for remote_write config %d. %s", i, err)
 			}
@@ -1000,24 +1014,22 @@ func (c *Operator) createOrUpdateConfigurationSecret(p *monitoringv1.Prometheus,
 	}
 
 	sClient := c.kclient.CoreV1().Secrets(p.Namespace)
+	SecretsInPromNS, err := sClient.List(metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
 
-	listSecrets, err := sClient.List(metav1.ListOptions{})
+	basicAuthSecrets, err := c.loadBasicAuthSecrets(smons, p.Spec.RemoteRead, p.Spec.RemoteWrite, SecretsInPromNS)
 
 	if err != nil {
 		return err
 	}
 
-	basicAuthSecrets, err := c.loadBasicAuthSecrets(smons, p.Spec.RemoteRead, p.Spec.RemoteWrite, listSecrets)
-
-	if err != nil {
-		return err
-	}
-
-	additionalScrapeConfigs, err := loadAdditionalScrapeConfigsSecret(p.Spec.AdditionalScrapeConfigs, listSecrets)
+	additionalScrapeConfigs, err := loadAdditionalScrapeConfigsSecret(p.Spec.AdditionalScrapeConfigs, SecretsInPromNS)
 	if err != nil {
 		return errors.Wrap(err, "loading additional scrape configs from Secret failed")
 	}
-	additionalAlertManagerConfigs, err := loadAdditionalScrapeConfigsSecret(p.Spec.AdditionalAlertManagerConfigs, listSecrets)
+	additionalAlertManagerConfigs, err := loadAdditionalScrapeConfigsSecret(p.Spec.AdditionalAlertManagerConfigs, SecretsInPromNS)
 	if err != nil {
 		return errors.Wrap(err, "loading additional alert manager configs from Secret failed")
 	}

test/basic-auth-test-app/Makefile

@@ -0,0 +1,13 @@
+REG := quay.io/coreos
+APP := basic-auth-test-app
+VERSION ?= $(shell cat VERSION)
+
+all: build push 
+
+build: 
+	@GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o basic-auth-test-app main.go
+	@docker build --build-arg VERSION=$(VERSION) -t $(REG)/$(APP):$(VERSION) --file $(APP).dockerfile .
+	@rm $(APP)
+
+push:
+	@docker push $(REG)/$(APP):$(VERSION)

test/basic-auth-test-app/VERSION

@@ -0,0 +1 @@
+0.1.0

test/basic-auth-test-app/basic-auth-test-app.dockerfile

@@ -0,0 +1,8 @@
+FROM alpine:3.7
+
+ARG VERSION="$VERSION"
+ENV VERSION="$VERSION"
+
+COPY basic-auth-test-app /
+
+ENTRYPOINT ["/basic-auth-test-app"]

test/basic-auth-test-app/main.go

@@ -0,0 +1,65 @@
+// Copyright 2016 The prometheus-operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+func main() {
+	http.HandleFunc("/", handler)
+	http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
+		if checkAuth(w, r) {
+			promhttp.Handler().ServeHTTP(w, r)
+			return
+		}
+
+		w.Header().Set("WWW-Authenticate", `Basic realm="MY REALM"`)
+		w.WriteHeader(401)
+		w.Write([]byte("401 Unauthorized\n"))
+	})
+
+	http.ListenAndServe(":8080", nil)
+}
+
+func handler(w http.ResponseWriter, r *http.Request) {
+	fmt.Fprintf(w, time.Now().String())
+	fmt.Fprintf(w, "\nAppVersion:"+os.Getenv("VERSION"))
+}
+
+func checkAuth(w http.ResponseWriter, r *http.Request) bool {
+	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
+	if len(s) != 2 {
+		return false
+	}
+
+	b, err := base64.StdEncoding.DecodeString(s[1])
+	if err != nil {
+		return false
+	}
+
+	pair := strings.SplitN(string(b), ":", 2)
+	if len(pair) != 2 {
+		return false
+	}
+
+	return pair[0] == "user" && pair[1] == "pass"
+}

test/e2e/prometheus_test.go

@@ -1253,6 +1253,106 @@ func TestThanos(t *testing.T) {
 	}
 }
 
+func TestPrometheusGetBasicAuthSecret(t *testing.T) {
+	t.Parallel()
+
+	ctx := framework.NewTestCtx(t)
+	defer ctx.Cleanup(t)
+	ns := ctx.CreateNamespace(t, framework.KubeClient)
+	ctx.SetupPrometheusRBACGlobal(t, ns, framework.KubeClient)
+
+	name := "test"
+
+	maptest := make(map[string]string)
+	maptest["tc"] = ns
+	prometheusCRD := framework.MakeBasicPrometheus(ns, name, name, 1)
+	prometheusCRD.Spec.ServiceMonitorNamespaceSelector = &metav1.LabelSelector{
+		MatchLabels: maptest,
+	}
+
+	if err := framework.CreatePrometheusAndWaitUntilReady(ns, prometheusCRD); err != nil {
+		t.Fatal(err)
+	}
+	testNamespace := ctx.CreateNamespace(t, framework.KubeClient)
+
+	testFramework.AddLabelsToNamespace(framework.KubeClient, testNamespace, maptest)
+
+	simple, err := testFramework.MakeDeployment("../../test/framework/ressources/basic-auth-app-deployment.yaml")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := testFramework.CreateDeployment(framework.KubeClient, testNamespace, simple); err != nil {
+		t.Fatal("Creating simple basic auth app failed: ", err)
+	}
+
+	authSecret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Data: map[string][]byte{
+			"user":     []byte("user"),
+			"password": []byte("pass"),
+		},
+	}
+
+	if _, err := framework.KubeClient.CoreV1().Secrets(testNamespace).Create(authSecret); err != nil {
+		t.Fatal(err)
+	}
+
+	svc := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+			Labels: map[string]string{
+				"group": name,
+			},
+		},
+		Spec: v1.ServiceSpec{
+			Type: v1.ServiceTypeLoadBalancer,
+			Ports: []v1.ServicePort{
+				v1.ServicePort{
+					Name: "web",
+					Port: 8080,
+				},
+			},
+			Selector: map[string]string{
+				"group": name,
+			},
+		},
+	}
+
+	sm := framework.MakeBasicServiceMonitor(name)
+	sm.Spec.Endpoints[0].BasicAuth = &monitoringv1.BasicAuth{
+		Username: v1.SecretKeySelector{
+			LocalObjectReference: v1.LocalObjectReference{
+				Name: name,
+			},
+			Key: "user",
+		},
+		Password: v1.SecretKeySelector{
+			LocalObjectReference: v1.LocalObjectReference{
+				Name: name,
+			},
+			Key: "password",
+		},
+	}
+
+	if finalizerFn, err := testFramework.CreateServiceAndWaitUntilReady(framework.KubeClient, testNamespace, svc); err != nil {
+		t.Fatal(err)
+	} else {
+		ctx.AddFinalizerFn(finalizerFn)
+	}
+
+	if _, err := framework.MonClientV1.ServiceMonitors(testNamespace).Create(sm); err != nil {
+		t.Fatal("Creating ServiceMonitor failed: ", err)
+	}
+
+	if err := framework.WaitForTargets(ns, "prometheus-operated", 1); err != nil {
+		t.Fatal(err)
+	}
+
+}
+
 func isDiscoveryWorking(ns, svcName, prometheusName string) func() (bool, error) {
 	return func() (bool, error) {
 		pods, err := framework.KubeClient.CoreV1().Pods(ns).List(prometheus.ListOptions(prometheusName))

test/framework/framework.go

@@ -202,6 +202,20 @@ func (ctx *TestCtx) SetupPrometheusRBAC(t *testing.T, ns string, kubeClient kube
 	}
 }
 
+func (ctx *TestCtx) SetupPrometheusRBACGlobal(t *testing.T, ns string, kubeClient kubernetes.Interface) {
+	if finalizerFn, err := CreateServiceAccount(kubeClient, ns, "../../example/rbac/prometheus/prometheus-service-account.yaml"); err != nil {
+		t.Fatal(errors.Wrap(err, "failed to create prometheus service account"))
+	} else {
+		ctx.AddFinalizerFn(finalizerFn)
+	}
+
+	if finalizerFn, err := CreateClusterRoleBinding(kubeClient, ns, "../../example/rbac/prometheus/prometheus-cluster-role-binding.yaml"); err != nil {
+		t.Fatal(errors.Wrap(err, "failed to create prometheus cluster role binding"))
+	} else {
+		ctx.AddFinalizerFn(finalizerFn)
+	}
+}
+
 // Teardown tears down a previously initialized test environment.
 func (f *Framework) Teardown() error {
 	if err := f.KubeClient.Core().Services(f.Namespace.Name).Delete("prometheus-operated", nil); err != nil && !k8sutil.IsResourceNotFoundError(err) {

test/framework/ressources/basic-auth-app-deployment.yaml

@@ -0,0 +1,23 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: basic-auth-test-app
+  labels:
+    group: test
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      group: test
+  template:
+    metadata:
+      labels:
+        group: test
+    spec:
+      containers:
+      - name: example-app
+        image: quay.io/coreos/basic-auth-test-app:0.1.0
+        imagePullPolicy: IfNotPresent
+        ports:
+        - name: web
+          containerPort: 8080