prometheus: Add test for additional scrape configs & update docs

Documentation/api.md

@@ -224,6 +224,7 @@ Specification of the desired behavior of the Prometheus cluster. More info: http
 | securityContext | SecurityContext holds pod-level security attributes and common container settings. This defaults to non root user with uid 1000 and gid 2000 for Prometheus >v2.0 and default PodSecurityContext for other versions. | *v1.PodSecurityContext | false |
 | listenLocal | ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP. | bool | false |
 | containers | Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. | []v1.Container | false |
+| additionalScrapeConfigs | AdditionalScrapeConfigs allows specifying a key of a Secret containing additional Prometheus scrape configurations. Scrape configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#<scrape_config>. As scrape configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible scrape configs are going to break Prometheus after the upgrade. | *[v1.SecretKeySelector](https://v1-6.docs.kubernetes.io/docs/api-reference/v1.6/#secretkeyselector-v1-core) | false |
 
 [Back to TOC](#table-of-contents)
 

Documentation/custom-configuration.md

@@ -3,6 +3,10 @@
     <i class="fa fa-exclamation-triangle"></i><b> Note:</b> Starting with v0.12.0, Prometheus Operator requires use of Kubernetes v1.7.x and up.
 </div>
 
+
+**Deprecation Warning:** The _custom configuration_ option of the Prometheus Operator will be deprecated in favor of the [_additional scrape config_](./api.md#prometheusspec) option.
+
+
 # Custom Configuration
 
 There are a few reasons, why one may want to provide a custom configuration to Prometheus instances, instead of having the Prometheus Operator generate the configuration based on `ServiceMonitor` objects.

example/prometheus-operator-crd/prometheus.crd.yaml

@@ -28,6 +28,21 @@ spec:
           description: 'Specification of the desired behavior of the Prometheus cluster.
             More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status'
           properties:
+            additionalScrapeConfigs:
+              description: SecretKeySelector selects a key of a Secret.
+              properties:
+                key:
+                  description: The key of the secret to select from.  Must be a valid
+                    secret key.
+                  type: string
+                name:
+                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                  type: string
+                optional:
+                  description: Specify whether the Secret or it's key must be defined
+                  type: boolean
+              required:
+              - key
             affinity:
               description: Affinity is a group of affinity scheduling rules.
               properties:

pkg/client/monitoring/v1/openapi_generated.go

@@ -917,11 +917,17 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
 								},
 							},
 						},
+						"additionalScrapeConfigs": {
+							SchemaProps: spec.SchemaProps{
+								Description: "AdditionalScrapeConfigs allows specifying a key of a Secret containing additional Prometheus scrape configurations. Scrape configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#<scrape_config>. As scrape configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible scrape configs are going to break Prometheus after the upgrade.",
+								Ref:         ref("k8s.io/api/core/v1.SecretKeySelector"),
+							},
+						},
 					},
 				},
 			},
 			Dependencies: []string{
-				"github.com/coreos/prometheus-operator/pkg/client/monitoring/v1.AlertingSpec", "github.com/coreos/prometheus-operator/pkg/client/monitoring/v1.RemoteReadSpec", "github.com/coreos/prometheus-operator/pkg/client/monitoring/v1.RemoteWriteSpec", "github.com/coreos/prometheus-operator/pkg/client/monitoring/v1.StorageSpec", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.Container", "k8s.io/api/core/v1.LocalObjectReference", "k8s.io/api/core/v1.PodSecurityContext", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.Toleration", "k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"},
+				"github.com/coreos/prometheus-operator/pkg/client/monitoring/v1.AlertingSpec", "github.com/coreos/prometheus-operator/pkg/client/monitoring/v1.RemoteReadSpec", "github.com/coreos/prometheus-operator/pkg/client/monitoring/v1.RemoteWriteSpec", "github.com/coreos/prometheus-operator/pkg/client/monitoring/v1.StorageSpec", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.Container", "k8s.io/api/core/v1.LocalObjectReference", "k8s.io/api/core/v1.PodSecurityContext", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecretKeySelector", "k8s.io/api/core/v1.Toleration", "k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"},
 		},
 		"github.com/coreos/prometheus-operator/pkg/client/monitoring/v1.PrometheusStatus": {
 			Schema: spec.Schema{

pkg/client/monitoring/v1/zz_generated.deepcopy.go

@@ -558,6 +558,15 @@ func (in *PrometheusSpec) DeepCopyInto(out *PrometheusSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.AdditionalScrapeConfigs != nil {
+		in, out := &in.AdditionalScrapeConfigs, &out.AdditionalScrapeConfigs
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(core_v1.SecretKeySelector)
+			(*in).DeepCopyInto(*out)
+		}
+	}
 	return
 }
 

pkg/prometheus/operator.go

@@ -898,10 +898,10 @@ func loadAdditionalScrapeConfigsSecret(additionalScrapeConfigs *v1.SecretKeySele
 					return c, nil
 				}
 
-				return nil, fmt.Errorf("Key %v could not be found in Secret %v.", additionalScrapeConfigs.Key, additionalScrapeConfigs.Name)
+				return nil, fmt.Errorf("key %v could not be found in Secret %v.", additionalScrapeConfigs.Key, additionalScrapeConfigs.Name)
 			}
 		}
-		return nil, fmt.Errorf("Secret %v could not be found.", additionalScrapeConfigs.Name)
+		return nil, fmt.Errorf("secret %v could not be found.", additionalScrapeConfigs.Name)
 	}
 	return nil, nil
 }

pkg/prometheus/promcfg.go

@@ -159,7 +159,7 @@ func generateConfig(p *v1.Prometheus, mons map[string]*v1.ServiceMonitor, ruleCo
 	var additionalScrapeConfigsYaml []yaml.MapSlice
 	err = yaml.Unmarshal([]byte(additionalScrapeConfigs), &additionalScrapeConfigsYaml)
 	if err != nil {
-		errors.Wrap(err, "Unmarshalling additional scrape configs failed")
+		errors.Wrap(err, "unmarshalling additional scrape configs failed")
 	}
 
 	cfg = append(cfg, yaml.MapItem{

pkg/prometheus/promcfg_test.go

@@ -106,6 +106,7 @@ func TestAlertmanagerBearerToken(t *testing.T) {
 		nil,
 		0,
 		map[string]BasicAuthCredentials{},
+		nil,
 	)
 	if err != nil {
 		t.Fatal(err)
@@ -276,6 +277,7 @@ func generateTestConfig(version string) ([]byte, error) {
 		makeServiceMonitors(),
 		1,
 		map[string]BasicAuthCredentials{},
+		nil,
 	)
 }
 

test/e2e/prometheus_test.go

@@ -267,6 +267,62 @@ scrape_configs:
 	}
 }
 
+func TestPrometheusAdditionalScrapeConfig(t *testing.T) {
+	ctx := framework.NewTestCtx(t)
+	defer ctx.Cleanup(t)
+	ns := ctx.CreateNamespace(t, framework.KubeClient)
+	ctx.SetupPrometheusRBAC(t, ns, framework.KubeClient)
+
+	prometheusName := "test"
+	group := "additional-config-test"
+	svc := framework.MakePrometheusService(prometheusName, group, v1.ServiceTypeClusterIP)
+
+	s := framework.MakeBasicServiceMonitor(group)
+	if _, err := framework.MonClient.ServiceMonitors(ns).Create(s); err != nil {
+		t.Fatal("Creating ServiceMonitor failed: ", err)
+	}
+
+	additionalConfig := `
+- job_name: "prometheus"
+  static_configs:
+  - targets: ["localhost:9090"]
+`
+	secret := v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "additional-scrape-configs",
+		},
+		Data: map[string][]byte{
+			"prometheus-additional.yaml": []byte(additionalConfig),
+		},
+	}
+	_, err := framework.KubeClient.CoreV1().Secrets(ns).Create(&secret)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	p := framework.MakeBasicPrometheus(ns, prometheusName, group, 1)
+	p.Spec.AdditionalScrapeConfigs = &v1.SecretKeySelector{
+		LocalObjectReference: v1.LocalObjectReference{
+			Name: "additional-scrape-configs",
+		},
+		Key: "prometheus-additional.yaml",
+	}
+	if err := framework.CreatePrometheusAndWaitUntilReady(ns, p); err != nil {
+		t.Fatal(err)
+	}
+
+	if finalizerFn, err := testFramework.CreateServiceAndWaitUntilReady(framework.KubeClient, ns, svc); err != nil {
+		t.Fatal(errors.Wrap(err, "creating prometheus service failed"))
+	} else {
+		ctx.AddFinalizerFn(finalizerFn)
+	}
+
+	// Wait for ServiceMonitor target, as well as additional-config target
+	if err := framework.WaitForTargets(ns, svc.Name, 2); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestPrometheusReloadRules(t *testing.T) {
 	t.Parallel()
 

test/framework/prometheus.go

@@ -260,7 +260,7 @@ func promImage(version string) string {
 func (f *Framework) WaitForTargets(ns, svcName string, amount int) error {
 	var targets []*Target
 
-	if err := wait.Poll(time.Second, time.Minute*10, func() (bool, error) {
+	if err := wait.Poll(time.Second, time.Minute*5, func() (bool, error) {
 		var err error
 		targets, err = f.GetActiveTargets(ns, svcName)
 		if err != nil {