prometheus: Allow specifying additional scrape configs via secret

example/additional-scrape-configs/additional-scrape-configs.yaml

@@ -0,0 +1,9 @@
+# this file was generated with the following command:
+# $ kubectl create secret generic additional-scrape-configs --from-file=prometheus-additional.yaml --dry-run -oyaml > additional-scrape-configs.yaml
+apiVersion: v1
+data:
+  prometheus-additional.yaml: LSBqb2JfbmFtZTogInByb21ldGhldXMiCiAgc3RhdGljX2NvbmZpZ3M6CiAgLSB0YXJnZXRzOiBbImxvY2FsaG9zdDo5MDkwIl0K
+kind: Secret
+metadata:
+  creationTimestamp: null
+  name: additional-scrape-configs

example/additional-scrape-configs/prometheus-additional.yaml

@@ -0,0 +1,3 @@
+- job_name: "prometheus"
+  static_configs:
+  - targets: ["localhost:9090"]

example/additional-scrape-configs/prometheus-cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus
+subjects:
+- kind: ServiceAccount
+  name: prometheus
+  namespace: default

example/additional-scrape-configs/prometheus-cluster-role.yaml

@@ -0,0 +1,18 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
+- apiGroups: [""]
+  resources:
+  - nodes
+  - services
+  - endpoints
+  - pods
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources:
+  - configmaps
+  verbs: ["get"]
+- nonResourceURLs: ["/metrics"]
+  verbs: ["get"]

example/additional-scrape-configs/prometheus-service-account.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus

example/additional-scrape-configs/prometheus.yaml

@@ -0,0 +1,23 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Prometheus
+metadata:
+  name: prometheus
+  labels:
+    prometheus: prometheus
+spec:
+  replicas: 2
+  serviceAccountName: prometheus
+  serviceMonitorSelector:
+    matchLabels:
+      team: frontend
+  alerting:
+    alertmanagers:
+    - namespace: default
+      name: alertmanager
+      port: web
+  resources:
+    requests:
+      memory: 400Mi
+  additionalScrapeConfigs:
+    name: additional-scrape-configs
+    key: prometheus-additional.yaml

pkg/client/monitoring/v1/types.go

@@ -132,6 +132,18 @@ type PrometheusSpec struct {
 	// Containers allows injecting additional containers. This is meant to
 	// allow adding an authentication proxy to a Prometheus pod.
 	Containers []v1.Container `json:"containers,omitempty"`
+	// AdditionalScrapeConfigs allows specifying a key of a Secret containing
+	// additional Prometheus scrape configurations. Scrape configurations
+	// specified are appended to the configurations generated by the Prometheus
+	// Operator. Job configurations specified must have the form as specified
+	// in the official Prometheus documentation:
+	// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#<scrape_config>.
+	// As scrape configs are appended, the user is responsible to make sure it
+	// is valid. Note that using this feature may expose the possibility to
+	// break upgrades of Prometheus. It is advised to review Prometheus release
+	// notes to ensure that no incompatible scrape configs are going to break
+	// Prometheus after the upgrade.
+	AdditionalScrapeConfigs *v1.SecretKeySelector `json:"additionalScrapeConfigs,omitempty"`
 }
 
 // Most recent observed status of the Prometheus cluster. Read-only. Not

pkg/prometheus/operator.go

@@ -890,6 +890,22 @@ func (c *Operator) destroyPrometheus(key string) error {
 	return nil
 }
 
+func loadAdditionalScrapeConfigsSecret(additionalScrapeConfigs *v1.SecretKeySelector, s *v1.SecretList) ([]byte, error) {
+	if additionalScrapeConfigs != nil {
+		for _, secret := range s.Items {
+			if secret.Name == additionalScrapeConfigs.Name {
+				if c, ok := secret.Data[additionalScrapeConfigs.Key]; ok {
+					return c, nil
+				}
+
+				return nil, fmt.Errorf("Key %v could not be found in Secret %v.", additionalScrapeConfigs.Key, additionalScrapeConfigs.Name)
+			}
+		}
+		return nil, fmt.Errorf("Secret %v could not be found.", additionalScrapeConfigs.Name)
+	}
+	return nil, nil
+}
+
 func loadBasicAuthSecret(basicAuth *monitoringv1.BasicAuth, s *v1.SecretList) (BasicAuthCredentials, error) {
 	var username string
 	var password string
@@ -992,8 +1008,13 @@ func (c *Operator) createConfig(p *monitoringv1.Prometheus, ruleFileConfigMaps [
 		return err
 	}
 
+	additionalScrapeConfigs, err := loadAdditionalScrapeConfigsSecret(p.Spec.AdditionalScrapeConfigs, listSecrets)
+	if err != nil {
+		return errors.Wrap(err, "loading additional scrape configs from Secret failed")
+	}
+
 	// Update secret based on the most recent configuration.
-	conf, err := generateConfig(p, smons, len(ruleFileConfigMaps), basicAuthSecrets)
+	conf, err := generateConfig(p, smons, len(ruleFileConfigMaps), basicAuthSecrets, additionalScrapeConfigs)
 	if err != nil {
 		return errors.Wrap(err, "generating config failed")
 	}

pkg/prometheus/promcfg.go

@@ -90,7 +90,7 @@ func buildExternalLabels(p *v1.Prometheus) yaml.MapSlice {
 	return stringMapToMapSlice(m)
 }
 
-func generateConfig(p *v1.Prometheus, mons map[string]*v1.ServiceMonitor, ruleConfigMaps int, basicAuthSecrets map[string]BasicAuthCredentials) ([]byte, error) {
+func generateConfig(p *v1.Prometheus, mons map[string]*v1.ServiceMonitor, ruleConfigMaps int, basicAuthSecrets map[string]BasicAuthCredentials, additionalScrapeConfigs []byte) ([]byte, error) {
 	versionStr := p.Spec.Version
 	if versionStr == "" {
 		versionStr = DefaultVersion
@@ -156,9 +156,15 @@ func generateConfig(p *v1.Prometheus, mons map[string]*v1.ServiceMonitor, ruleCo
 		}
 	}
 
+	var additionalScrapeConfigsYaml []yaml.MapSlice
+	err = yaml.Unmarshal([]byte(additionalScrapeConfigs), &additionalScrapeConfigsYaml)
+	if err != nil {
+		errors.Wrap(err, "Unmarshalling additional scrape configs failed")
+	}
+
 	cfg = append(cfg, yaml.MapItem{
 		Key:   "scrape_configs",
-		Value: scrapeConfigs,
+		Value: append(scrapeConfigs, additionalScrapeConfigsYaml...),
 	})
 
 	var alertRelabelConfigs []yaml.MapSlice