Merge pull request #1338 from WanderaOrg/additional-am-config

prometheus: Allow specifying additional AlertManager configs via secret

Documentation/api.md

@@ -225,6 +225,7 @@ Specification of the desired behavior of the Prometheus cluster. More info: http
 | listenLocal | ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP. | bool | false |
 | containers | Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. | []v1.Container | false |
 | additionalScrapeConfigs | AdditionalScrapeConfigs allows specifying a key of a Secret containing additional Prometheus scrape configurations. Scrape configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#<scrape_config>. As scrape configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible scrape configs are going to break Prometheus after the upgrade. | *[v1.SecretKeySelector](https://v1-6.docs.kubernetes.io/docs/api-reference/v1.6/#secretkeyselector-v1-core) | false |
+| additionalAlertManagerConfigs | AdditionalAlertManagerConfigs allows specifying a key of a Secret containing additional Prometheus AlertManager configurations. AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#<alertmanager_config>. As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade. | *[v1.SecretKeySelector](https://v1-6.docs.kubernetes.io/docs/api-reference/v1.6/#secretkeyselector-v1-core) | false |
 
 [Back to TOC](#table-of-contents)
 

example/prometheus-operator-crd/prometheus.crd.yaml

@@ -28,6 +28,21 @@ spec:
           description: 'Specification of the desired behavior of the Prometheus cluster.
             More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status'
           properties:
+            additionalAlertManagerConfigs:
+              description: SecretKeySelector selects a key of a Secret.
+              properties:
+                key:
+                  description: The key of the secret to select from.  Must be a valid
+                    secret key.
+                  type: string
+                name:
+                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                  type: string
+                optional:
+                  description: Specify whether the Secret or it's key must be defined
+                  type: boolean
+              required:
+              - key
             additionalScrapeConfigs:
               description: SecretKeySelector selects a key of a Secret.
               properties:

pkg/client/monitoring/v1/openapi_generated.go

@@ -917,6 +917,12 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
 								Ref:         ref("k8s.io/api/core/v1.SecretKeySelector"),
 							},
 						},
+						"additionalAlertManagerConfigs": {
+							SchemaProps: spec.SchemaProps{
+								Description: "AdditionalAlertManagerConfigs allows specifying a key of a Secret containing additional Prometheus AlertManager configurations. AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#<alertmanager_config>. As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade.",
+								Ref:         ref("k8s.io/api/core/v1.SecretKeySelector"),
+							},
+						},
 					},
 				},
 			},

pkg/client/monitoring/v1/types.go

@@ -147,6 +147,18 @@ type PrometheusSpec struct {
 	// notes to ensure that no incompatible scrape configs are going to break
 	// Prometheus after the upgrade.
 	AdditionalScrapeConfigs *v1.SecretKeySelector `json:"additionalScrapeConfigs,omitempty"`
+	// AdditionalAlertManagerConfigs allows specifying a key of a Secret containing
+	// additional Prometheus AlertManager configurations. AlertManager configurations
+	// specified are appended to the configurations generated by the Prometheus
+	// Operator. Job configurations specified must have the form as specified
+	// in the official Prometheus documentation:
+	// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#<alertmanager_config>.
+	// As AlertManager configs are appended, the user is responsible to make sure it
+	// is valid. Note that using this feature may expose the possibility to
+	// break upgrades of Prometheus. It is advised to review Prometheus release
+	// notes to ensure that no incompatible AlertManager configs are going to break
+	// Prometheus after the upgrade.
+	AdditionalAlertManagerConfigs *v1.SecretKeySelector `json:"additionalAlertManagerConfigs,omitempty"`
 }
 
 // Most recent observed status of the Prometheus cluster. Read-only. Not

pkg/client/monitoring/v1/zz_generated.deepcopy.go

@@ -571,6 +571,15 @@ func (in *PrometheusSpec) DeepCopyInto(out *PrometheusSpec) {
 			(*in).DeepCopyInto(*out)
 		}
 	}
+	if in.AdditionalAlertManagerConfigs != nil {
+		in, out := &in.AdditionalAlertManagerConfigs, &out.AdditionalAlertManagerConfigs
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(core_v1.SecretKeySelector)
+			(*in).DeepCopyInto(*out)
+		}
+	}
 	return
 }
 

pkg/prometheus/operator.go

@@ -899,18 +899,18 @@ func (c *Operator) destroyPrometheus(key string) error {
 	return nil
 }
 
-func loadAdditionalScrapeConfigsSecret(additionalScrapeConfigs *v1.SecretKeySelector, s *v1.SecretList) ([]byte, error) {
-	if additionalScrapeConfigs != nil {
+func loadAdditionalConfigsSecret(additionalConfigs *v1.SecretKeySelector, s *v1.SecretList) ([]byte, error) {
+	if additionalConfigs != nil {
 		for _, secret := range s.Items {
-			if secret.Name == additionalScrapeConfigs.Name {
-				if c, ok := secret.Data[additionalScrapeConfigs.Key]; ok {
+			if secret.Name == additionalConfigs.Name {
+				if c, ok := secret.Data[additionalConfigs.Key]; ok {
 					return c, nil
 				}
 
-				return nil, fmt.Errorf("key %v could not be found in Secret %v.", additionalScrapeConfigs.Key, additionalScrapeConfigs.Name)
+				return nil, fmt.Errorf("key %v could not be found in Secret %v.", additionalConfigs.Key, additionalConfigs.Name)
 			}
 		}
-		return nil, fmt.Errorf("secret %v could not be found.", additionalScrapeConfigs.Name)
+		return nil, fmt.Errorf("secret %v could not be found.", additionalConfigs.Name)
 	}
 	return nil, nil
 }
@@ -1017,13 +1017,17 @@ func (c *Operator) createConfig(p *monitoringv1.Prometheus, ruleFileConfigMaps [
 		return err
 	}
 
-	additionalScrapeConfigs, err := loadAdditionalScrapeConfigsSecret(p.Spec.AdditionalScrapeConfigs, listSecrets)
+	additionalScrapeConfigs, err := loadAdditionalConfigsSecret(p.Spec.AdditionalScrapeConfigs, listSecrets)
 	if err != nil {
 		return errors.Wrap(err, "loading additional scrape configs from Secret failed")
 	}
+	additionalAlertManagerConfigs, err := loadAdditionalConfigsSecret(p.Spec.AdditionalAlertManagerConfigs, listSecrets)
+	if err != nil {
+		return errors.Wrap(err, "loading additional alert manager configs from Secret failed")
+	}
 
 	// Update secret based on the most recent configuration.
-	conf, err := generateConfig(p, smons, len(ruleFileConfigMaps), basicAuthSecrets, additionalScrapeConfigs)
+	conf, err := generateConfig(p, smons, len(ruleFileConfigMaps), basicAuthSecrets, additionalScrapeConfigs, additionalAlertManagerConfigs)
 	if err != nil {
 		return errors.Wrap(err, "generating config failed")
 	}

pkg/prometheus/promcfg.go

@@ -90,7 +90,7 @@ func buildExternalLabels(p *v1.Prometheus) yaml.MapSlice {
 	return stringMapToMapSlice(m)
 }
 
-func generateConfig(p *v1.Prometheus, mons map[string]*v1.ServiceMonitor, ruleConfigMaps int, basicAuthSecrets map[string]BasicAuthCredentials, additionalScrapeConfigs []byte) ([]byte, error) {
+func generateConfig(p *v1.Prometheus, mons map[string]*v1.ServiceMonitor, ruleConfigMaps int, basicAuthSecrets map[string]BasicAuthCredentials, additionalScrapeConfigs []byte, additionalAlertManagerConfigs []byte) ([]byte, error) {
 	versionStr := p.Spec.Version
 	if versionStr == "" {
 		versionStr = DefaultVersion
@@ -167,6 +167,14 @@ func generateConfig(p *v1.Prometheus, mons map[string]*v1.ServiceMonitor, ruleCo
 		Value: append(scrapeConfigs, additionalScrapeConfigsYaml...),
 	})
 
+	var additionalAlertManagerConfigsYaml []yaml.MapSlice
+	err = yaml.Unmarshal([]byte(additionalAlertManagerConfigs), &additionalAlertManagerConfigsYaml)
+	if err != nil {
+		errors.Wrap(err, "unmarshalling additional alert manager configs failed")
+	}
+
+	alertmanagerConfigs = append(alertmanagerConfigs, additionalAlertManagerConfigsYaml...)
+
 	var alertRelabelConfigs []yaml.MapSlice
 
 	// action 'labeldrop' is not supported <= v1.4.1

pkg/prometheus/promcfg_test.go

@@ -107,6 +107,7 @@ func TestAlertmanagerBearerToken(t *testing.T) {
 		0,
 		map[string]BasicAuthCredentials{},
 		nil,
+		nil,
 	)
 	if err != nil {
 		t.Fatal(err)
@@ -204,6 +205,7 @@ func generateTestConfig(version string) ([]byte, error) {
 		1,
 		map[string]BasicAuthCredentials{},
 		nil,
+		nil,
 	)
 }
 

test/e2e/prometheus_test.go

@@ -324,6 +324,85 @@ func TestPrometheusAdditionalScrapeConfig(t *testing.T) {
 	}
 }
 
+func TestPrometheusAdditionalAlertManagerConfig(t *testing.T) {
+	ctx := framework.NewTestCtx(t)
+	defer ctx.Cleanup(t)
+	ns := ctx.CreateNamespace(t, framework.KubeClient)
+	ctx.SetupPrometheusRBAC(t, ns, framework.KubeClient)
+
+	prometheusName := "test"
+	group := "additional-alert-config-test"
+	svc := framework.MakePrometheusService(prometheusName, group, v1.ServiceTypeClusterIP)
+
+	s := framework.MakeBasicServiceMonitor(group)
+	if _, err := framework.MonClient.ServiceMonitors(ns).Create(s); err != nil {
+		t.Fatal("Creating ServiceMonitor failed: ", err)
+	}
+
+	additionalConfig := `
+- path_prefix: /
+  scheme: http
+  static_configs:
+  - targets: ["localhost:9093"]
+`
+	secret := v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "additional-alert-configs",
+		},
+		Data: map[string][]byte{
+			"prometheus-additional.yaml": []byte(additionalConfig),
+		},
+	}
+	_, err := framework.KubeClient.CoreV1().Secrets(ns).Create(&secret)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	p := framework.MakeBasicPrometheus(ns, prometheusName, group, 1)
+	p.Spec.AdditionalAlertManagerConfigs = &v1.SecretKeySelector{
+		LocalObjectReference: v1.LocalObjectReference{
+			Name: "additional-alert-configs",
+		},
+		Key: "prometheus-additional.yaml",
+	}
+	if err := framework.CreatePrometheusAndWaitUntilReady(ns, p); err != nil {
+		t.Fatal(err)
+	}
+
+	if finalizerFn, err := testFramework.CreateServiceAndWaitUntilReady(framework.KubeClient, ns, svc); err != nil {
+		t.Fatal(errors.Wrap(err, "creating prometheus service failed"))
+	} else {
+		ctx.AddFinalizerFn(finalizerFn)
+	}
+
+	// Wait for ServiceMonitor target
+	if err := framework.WaitForTargets(ns, svc.Name, 1); err != nil {
+		t.Fatal(err)
+	}
+
+	err = wait.Poll(time.Second, 5*time.Minute, func() (done bool, err error) {
+		response, err := framework.QueryPrometheusSVC(ns, svc.Name, "/api/v1/alertmanagers", map[string]string{})
+		if err != nil {
+			return true, err
+		}
+
+		ra := prometheusAlertmanagerAPIResponse{}
+		if err := json.NewDecoder(bytes.NewBuffer(response)).Decode(&ra); err != nil {
+			return true, err
+		}
+
+		if ra.Status == "success" && len(ra.Data.ActiveAlertmanagers) == 1 {
+			return true, nil
+		}
+
+		return false, nil
+	})
+
+	if err != nil {
+		t.Fatal(errors.Wrap(err, "validating Prometheus Alertmanager configuration failed"))
+	}
+}
+
 func TestPrometheusReloadRules(t *testing.T) {
 	t.Parallel()