From 982277b0f218e27ded2d7b97c2b15ee70632d7fc Mon Sep 17 00:00:00 2001 From: Huy Le Date: Mon, 23 Jul 2018 10:50:12 -0700 Subject: [PATCH] prometheus: Load basic-auth secrets from ServiceMonitor namespace (#1619) Basic-Auth Secrets for ServiceMonitor used to be loaded from the prometheus'namespace. With this change, Prometheus will load the secret from the servicemonitor namespace. The commit also includes a basic-auth image for the integration test --- pkg/prometheus/operator.go | 38 ++++--- test/basic-auth-test-app/Makefile | 13 +++ test/basic-auth-test-app/VERSION | 1 + .../basic-auth-test-app.dockerfile | 8 ++ test/basic-auth-test-app/main.go | 65 ++++++++++++ test/e2e/prometheus_test.go | 100 ++++++++++++++++++ test/framework/framework.go | 14 +++ .../ressources/basic-auth-app-deployment.yaml | 23 ++++ 8 files changed, 249 insertions(+), 13 deletions(-) create mode 100644 test/basic-auth-test-app/Makefile create mode 100644 test/basic-auth-test-app/VERSION create mode 100644 test/basic-auth-test-app/basic-auth-test-app.dockerfile create mode 100644 test/basic-auth-test-app/main.go create mode 100644 test/framework/ressources/basic-auth-app-deployment.yaml diff --git a/pkg/prometheus/operator.go b/pkg/prometheus/operator.go index 50d19b9ff..659253832 100644 --- a/pkg/prometheus/operator.go +++ b/pkg/prometheus/operator.go @@ -955,12 +955,26 @@ func loadBasicAuthSecret(basicAuth *monitoringv1.BasicAuth, s *v1.SecretList) (B } -func (c *Operator) loadBasicAuthSecrets(mons map[string]*monitoringv1.ServiceMonitor, remoteReads []monitoringv1.RemoteReadSpec, remoteWrites []monitoringv1.RemoteWriteSpec, s *v1.SecretList) (map[string]BasicAuthCredentials, error) { +func (c *Operator) loadBasicAuthSecrets(mons map[string]*monitoringv1.ServiceMonitor, remoteReads []monitoringv1.RemoteReadSpec, remoteWrites []monitoringv1.RemoteWriteSpec, SecretsInPromNS *v1.SecretList) (map[string]BasicAuthCredentials, error) { + sMonSecretMap := make(map[string]*v1.SecretList) + for _, mon := range mons { + smNamespace := mon.Namespace + if sMonSecretMap[smNamespace] == nil { + msClient := c.kclient.CoreV1().Secrets(smNamespace) + listSecrets, err := msClient.List(metav1.ListOptions{}) + + if err != nil { + return nil, errors.Wrapf(err, "failed to retrieve secrets in namespace '%v' for servicemonitor '%v'", smNamespace, mon.Name) + } + sMonSecretMap[smNamespace] = listSecrets + } + } + secrets := map[string]BasicAuthCredentials{} for _, mon := range mons { for i, ep := range mon.Spec.Endpoints { if ep.BasicAuth != nil { - credentials, err := loadBasicAuthSecret(ep.BasicAuth, s) + credentials, err := loadBasicAuthSecret(ep.BasicAuth, sMonSecretMap[mon.Namespace]) if err != nil { return nil, fmt.Errorf("could not generate basicAuth for servicemonitor %s. %s", mon.Name, err) } @@ -971,7 +985,7 @@ func (c *Operator) loadBasicAuthSecrets(mons map[string]*monitoringv1.ServiceMon for i, remote := range remoteReads { if remote.BasicAuth != nil { - credentials, err := loadBasicAuthSecret(remote.BasicAuth, s) + credentials, err := loadBasicAuthSecret(remote.BasicAuth, SecretsInPromNS) if err != nil { return nil, fmt.Errorf("could not generate basicAuth for remote_read config %d. %s", i, err) } @@ -981,7 +995,7 @@ func (c *Operator) loadBasicAuthSecrets(mons map[string]*monitoringv1.ServiceMon for i, remote := range remoteWrites { if remote.BasicAuth != nil { - credentials, err := loadBasicAuthSecret(remote.BasicAuth, s) + credentials, err := loadBasicAuthSecret(remote.BasicAuth, SecretsInPromNS) if err != nil { return nil, fmt.Errorf("could not generate basicAuth for remote_write config %d. %s", i, err) } @@ -1000,24 +1014,22 @@ func (c *Operator) createOrUpdateConfigurationSecret(p *monitoringv1.Prometheus, } sClient := c.kclient.CoreV1().Secrets(p.Namespace) + SecretsInPromNS, err := sClient.List(metav1.ListOptions{}) + if err != nil { + return err + } - listSecrets, err := sClient.List(metav1.ListOptions{}) + basicAuthSecrets, err := c.loadBasicAuthSecrets(smons, p.Spec.RemoteRead, p.Spec.RemoteWrite, SecretsInPromNS) if err != nil { return err } - basicAuthSecrets, err := c.loadBasicAuthSecrets(smons, p.Spec.RemoteRead, p.Spec.RemoteWrite, listSecrets) - - if err != nil { - return err - } - - additionalScrapeConfigs, err := loadAdditionalScrapeConfigsSecret(p.Spec.AdditionalScrapeConfigs, listSecrets) + additionalScrapeConfigs, err := loadAdditionalScrapeConfigsSecret(p.Spec.AdditionalScrapeConfigs, SecretsInPromNS) if err != nil { return errors.Wrap(err, "loading additional scrape configs from Secret failed") } - additionalAlertManagerConfigs, err := loadAdditionalScrapeConfigsSecret(p.Spec.AdditionalAlertManagerConfigs, listSecrets) + additionalAlertManagerConfigs, err := loadAdditionalScrapeConfigsSecret(p.Spec.AdditionalAlertManagerConfigs, SecretsInPromNS) if err != nil { return errors.Wrap(err, "loading additional alert manager configs from Secret failed") } diff --git a/test/basic-auth-test-app/Makefile b/test/basic-auth-test-app/Makefile new file mode 100644 index 000000000..ef6b0c67b --- /dev/null +++ b/test/basic-auth-test-app/Makefile @@ -0,0 +1,13 @@ +REG := quay.io/coreos +APP := basic-auth-test-app +VERSION ?= $(shell cat VERSION) + +all: build push + +build: + @GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o basic-auth-test-app main.go + @docker build --build-arg VERSION=$(VERSION) -t $(REG)/$(APP):$(VERSION) --file $(APP).dockerfile . + @rm $(APP) + +push: + @docker push $(REG)/$(APP):$(VERSION) diff --git a/test/basic-auth-test-app/VERSION b/test/basic-auth-test-app/VERSION new file mode 100644 index 000000000..6e8bf73aa --- /dev/null +++ b/test/basic-auth-test-app/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/test/basic-auth-test-app/basic-auth-test-app.dockerfile b/test/basic-auth-test-app/basic-auth-test-app.dockerfile new file mode 100644 index 000000000..914221a4f --- /dev/null +++ b/test/basic-auth-test-app/basic-auth-test-app.dockerfile @@ -0,0 +1,8 @@ +FROM alpine:3.7 + +ARG VERSION="$VERSION" +ENV VERSION="$VERSION" + +COPY basic-auth-test-app / + +ENTRYPOINT ["/basic-auth-test-app"] diff --git a/test/basic-auth-test-app/main.go b/test/basic-auth-test-app/main.go new file mode 100644 index 000000000..9220ba4a2 --- /dev/null +++ b/test/basic-auth-test-app/main.go @@ -0,0 +1,65 @@ +// Copyright 2016 The prometheus-operator Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/base64" + "fmt" + "github.com/prometheus/client_golang/prometheus/promhttp" + "net/http" + "os" + "strings" + "time" +) + +func main() { + http.HandleFunc("/", handler) + http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) { + if checkAuth(w, r) { + promhttp.Handler().ServeHTTP(w, r) + return + } + + w.Header().Set("WWW-Authenticate", `Basic realm="MY REALM"`) + w.WriteHeader(401) + w.Write([]byte("401 Unauthorized\n")) + }) + + http.ListenAndServe(":8080", nil) +} + +func handler(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, time.Now().String()) + fmt.Fprintf(w, "\nAppVersion:"+os.Getenv("VERSION")) +} + +func checkAuth(w http.ResponseWriter, r *http.Request) bool { + s := strings.SplitN(r.Header.Get("Authorization"), " ", 2) + if len(s) != 2 { + return false + } + + b, err := base64.StdEncoding.DecodeString(s[1]) + if err != nil { + return false + } + + pair := strings.SplitN(string(b), ":", 2) + if len(pair) != 2 { + return false + } + + return pair[0] == "user" && pair[1] == "pass" +} diff --git a/test/e2e/prometheus_test.go b/test/e2e/prometheus_test.go index 8f3c63788..b9a9ea8a1 100644 --- a/test/e2e/prometheus_test.go +++ b/test/e2e/prometheus_test.go @@ -1253,6 +1253,106 @@ func TestThanos(t *testing.T) { } } +func TestPrometheusGetBasicAuthSecret(t *testing.T) { + t.Parallel() + + ctx := framework.NewTestCtx(t) + defer ctx.Cleanup(t) + ns := ctx.CreateNamespace(t, framework.KubeClient) + ctx.SetupPrometheusRBACGlobal(t, ns, framework.KubeClient) + + name := "test" + + maptest := make(map[string]string) + maptest["tc"] = ns + prometheusCRD := framework.MakeBasicPrometheus(ns, name, name, 1) + prometheusCRD.Spec.ServiceMonitorNamespaceSelector = &metav1.LabelSelector{ + MatchLabels: maptest, + } + + if err := framework.CreatePrometheusAndWaitUntilReady(ns, prometheusCRD); err != nil { + t.Fatal(err) + } + testNamespace := ctx.CreateNamespace(t, framework.KubeClient) + + testFramework.AddLabelsToNamespace(framework.KubeClient, testNamespace, maptest) + + simple, err := testFramework.MakeDeployment("../../test/framework/ressources/basic-auth-app-deployment.yaml") + if err != nil { + t.Fatal(err) + } + + if err := testFramework.CreateDeployment(framework.KubeClient, testNamespace, simple); err != nil { + t.Fatal("Creating simple basic auth app failed: ", err) + } + + authSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Data: map[string][]byte{ + "user": []byte("user"), + "password": []byte("pass"), + }, + } + + if _, err := framework.KubeClient.CoreV1().Secrets(testNamespace).Create(authSecret); err != nil { + t.Fatal(err) + } + + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "group": name, + }, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + Ports: []v1.ServicePort{ + v1.ServicePort{ + Name: "web", + Port: 8080, + }, + }, + Selector: map[string]string{ + "group": name, + }, + }, + } + + sm := framework.MakeBasicServiceMonitor(name) + sm.Spec.Endpoints[0].BasicAuth = &monitoringv1.BasicAuth{ + Username: v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: name, + }, + Key: "user", + }, + Password: v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: name, + }, + Key: "password", + }, + } + + if finalizerFn, err := testFramework.CreateServiceAndWaitUntilReady(framework.KubeClient, testNamespace, svc); err != nil { + t.Fatal(err) + } else { + ctx.AddFinalizerFn(finalizerFn) + } + + if _, err := framework.MonClientV1.ServiceMonitors(testNamespace).Create(sm); err != nil { + t.Fatal("Creating ServiceMonitor failed: ", err) + } + + if err := framework.WaitForTargets(ns, "prometheus-operated", 1); err != nil { + t.Fatal(err) + } + +} + func isDiscoveryWorking(ns, svcName, prometheusName string) func() (bool, error) { return func() (bool, error) { pods, err := framework.KubeClient.CoreV1().Pods(ns).List(prometheus.ListOptions(prometheusName)) diff --git a/test/framework/framework.go b/test/framework/framework.go index c194b9db1..05a6d369e 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -202,6 +202,20 @@ func (ctx *TestCtx) SetupPrometheusRBAC(t *testing.T, ns string, kubeClient kube } } +func (ctx *TestCtx) SetupPrometheusRBACGlobal(t *testing.T, ns string, kubeClient kubernetes.Interface) { + if finalizerFn, err := CreateServiceAccount(kubeClient, ns, "../../example/rbac/prometheus/prometheus-service-account.yaml"); err != nil { + t.Fatal(errors.Wrap(err, "failed to create prometheus service account")) + } else { + ctx.AddFinalizerFn(finalizerFn) + } + + if finalizerFn, err := CreateClusterRoleBinding(kubeClient, ns, "../../example/rbac/prometheus/prometheus-cluster-role-binding.yaml"); err != nil { + t.Fatal(errors.Wrap(err, "failed to create prometheus cluster role binding")) + } else { + ctx.AddFinalizerFn(finalizerFn) + } +} + // Teardown tears down a previously initialized test environment. func (f *Framework) Teardown() error { if err := f.KubeClient.Core().Services(f.Namespace.Name).Delete("prometheus-operated", nil); err != nil && !k8sutil.IsResourceNotFoundError(err) { diff --git a/test/framework/ressources/basic-auth-app-deployment.yaml b/test/framework/ressources/basic-auth-app-deployment.yaml new file mode 100644 index 000000000..237973bd7 --- /dev/null +++ b/test/framework/ressources/basic-auth-app-deployment.yaml @@ -0,0 +1,23 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: basic-auth-test-app + labels: + group: test +spec: + replicas: 1 + selector: + matchLabels: + group: test + template: + metadata: + labels: + group: test + spec: + containers: + - name: example-app + image: quay.io/coreos/basic-auth-test-app:0.1.0 + imagePullPolicy: IfNotPresent + ports: + - name: web + containerPort: 8080