ServiceMonitor: Enable Prometheus to select ServMon outside own ns

So far a Prometheus object could only select ServiceMonitors inside its
own namespace. This patch enables a Prometheus object to select
ServiceMonitors outside its own namespace via the
`ServiceMonitorNamespaceSelector` field in the Prometheus spec.

Use case: There is one Prometheus inside the `monitoring` namespace,
which is supposed to monitor applications across namespaces for an
entire Kubernetes cluster.  Each app team is supposed to manage its own
ServiceMonitors. Instead of granting each app team access to the
`monitoring` namespace to manage its ServiceMonitor objects,
ServiceMonitors can be shipped along with the application itself in each
application namespace.

Documentation/api.md

@@ -198,6 +198,7 @@ Specification of the desired behavior of the Prometheus cluster. More info: http
 | ----- | ----------- | ------ | -------- |
 | podMetadata | Standard object’s metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#metadata Metadata Labels and Annotations gets propagated to the prometheus pods. | *[metav1.ObjectMeta](https://v1-6.docs.kubernetes.io/docs/api-reference/v1.6/#objectmeta-v1-meta) | false |
 | serviceMonitorSelector | ServiceMonitors to be selected for target discovery. | *[metav1.LabelSelector](https://v1-6.docs.kubernetes.io/docs/api-reference/v1.6/#labelselector-v1-meta) | false |
+| serviceMonitorNamespaceSelector | Namespaces to be selected for ServiceMonitor discovery. If empty, only check own namespace. | *[metav1.LabelSelector](https://v1-6.docs.kubernetes.io/docs/api-reference/v1.6/#labelselector-v1-meta) | false |
 | version | Version of Prometheus to be deployed. | string | false |
 | paused | When a Prometheus deployment is paused, no actions except for deletion will be performed on the underlying objects. | bool | false |
 | baseImage | Base image to use for a Prometheus deployment. | string | false |

Documentation/design.md

@@ -37,7 +37,7 @@ The `endpoints` section of the `ServiceMonitorSpec`, is used to configure which
 
 > Note: `endpoints` (lowercase) is the field in the `ServiceMonitor` CRD, while `Endpoints` (capitalized) is the Kubernetes object kind.
 
-While `ServiceMonitor`s must live in the same namespace as the `Prometheus` resource, discovered targets may come from any namespace. This is important to allow cross-namespace monitoring use cases, e.g. for meta-monitoring. Using the `namespaceSelector` of the `ServiceMonitorSpec`, one can restrict the namespaces the `Endpoints` objects are allowed to be discovered from.
+Both `ServiceMonitors` as well as discovered targets may come from any namespace. This is important to allow cross-namespace monitoring use cases, e.g. for meta-monitoring. Using the `ServiceMonitorNamespaceSelector` of the `PrometheusSpec`, one can restrict the namespaces `ServiceMonitor`s are selected from by the respective Prometheus server. Using the `namespaceSelector` of the `ServiceMonitorSpec`, one can restrict the namespaces the `Endpoints` objects are allowed to be discovered from.
 To discover targets in all namespaces the `namespaceSelector` has to be empty:
 ```yaml
 spec:

Documentation/rbac.md

@@ -64,11 +64,8 @@ rules:
 - apiGroups: [""]
   resources:
   - nodes
-  verbs: ["list", "watch"]
-- apiGroups: [""]
-  resources:
   - namespaces
-  verbs: ["list"]
+  verbs: ["list", "watch"]
 ```
 
 > Note: A cluster admin is required to create this `ClusterRole` and create a `ClusterRoleBinding` or `RoleBinding` to the `ServiceAccount` used by the Prometheus Operator `Pod`. The `ServiceAccount` used by the Prometheus Operator `Pod` can be specified in the `Deployment` object used to deploy it.

Documentation/user-guides/getting-started.md

@@ -78,11 +78,8 @@ rules:
 - apiGroups: [""]
   resources:
   - nodes
-  verbs: ["list", "watch"]
-- apiGroups: [""]
-  resources:
   - namespaces
-  verbs: ["list"]
+  verbs: ["list", "watch"]
 ---
 apiVersion: v1
 kind: ServiceAccount

bundle.yaml

@@ -60,11 +60,8 @@ rules:
 - apiGroups: [""]
   resources:
   - nodes
-  verbs: ["list", "watch"]
-- apiGroups: [""]
-  resources:
   - namespaces
-  verbs: ["list"]
+  verbs: ["list", "watch"]
 ---
 apiVersion: v1
 kind: ServiceAccount

example/prometheus-operator-crd/prometheus.crd.yaml

@@ -1983,6 +1983,48 @@ spec:
               description: ServiceAccountName is the name of the ServiceAccount to
                 use to run the Prometheus Pods.
               type: string
+            serviceMonitorNamespaceSelector:
+              description: A label selector is a label query over a set of resources.
+                The result of matchLabels and matchExpressions are ANDed. An empty
+                label selector matches all objects. A null label selector matches
+                no objects.
+              properties:
+                matchExpressions:
+                  description: matchExpressions is a list of label selector requirements.
+                    The requirements are ANDed.
+                  items:
+                    description: A label selector requirement is a selector that contains
+                      values, a key, and an operator that relates the key and values.
+                    properties:
+                      key:
+                        description: key is the label key that the selector applies
+                          to.
+                        type: string
+                      operator:
+                        description: operator represents a key's relationship to a
+                          set of values. Valid operators are In, NotIn, Exists and
+                          DoesNotExist.
+                        type: string
+                      values:
+                        description: values is an array of string values. If the operator
+                          is In or NotIn, the values array must be non-empty. If the
+                          operator is Exists or DoesNotExist, the values array must
+                          be empty. This array is replaced during a strategic merge
+                          patch.
+                        items:
+                          type: string
+                        type: array
+                    required:
+                    - key
+                    - operator
+                  type: array
+                matchLabels:
+                  description: matchLabels is a map of {key,value} pairs. A single
+                    {key,value} in the matchLabels map is equivalent to an element
+                    of matchExpressions, whose key field is "key", the operator is
+                    "In", and the values array contains only "value". The requirements
+                    are ANDed.
+                  type: object
             serviceMonitorSelector:
               description: A label selector is a label query over a set of resources.
                 The result of matchLabels and matchExpressions are ANDed. An empty

example/rbac/prometheus-operator/prometheus-operator-cluster-role.yaml

@@ -47,8 +47,5 @@ rules:
 - apiGroups: [""]
   resources:
   - nodes
-  verbs: ["list", "watch"]
-- apiGroups: [""]
-  resources:
   - namespaces
-  verbs: ["list"]
+  verbs: ["list", "watch"]

pkg/client/monitoring/v1/openapi_generated.go

@@ -690,6 +690,12 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
 								Ref:         ref("k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"),
 							},
 						},
+						"serviceMonitorNamespaceSelector": {
+							SchemaProps: spec.SchemaProps{
+								Description: "Namespaces to be selected for ServiceMonitor discovery. If empty, only check own namespace.",
+								Ref:         ref("k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"),
+							},
+						},
 						"version": {
 							SchemaProps: spec.SchemaProps{
 								Description: "Version of Prometheus to be deployed.",

pkg/client/monitoring/v1/types.go

@@ -60,6 +60,9 @@ type PrometheusSpec struct {
 	PodMetadata *metav1.ObjectMeta `json:"podMetadata,omitempty"`
 	// ServiceMonitors to be selected for target discovery.
 	ServiceMonitorSelector *metav1.LabelSelector `json:"serviceMonitorSelector,omitempty"`
+	// Namespaces to be selected for ServiceMonitor discovery. If empty, only
+	// check own namespace.
+	ServiceMonitorNamespaceSelector *metav1.LabelSelector `json:"serviceMonitorNamespaceSelector,omitempty"`
 	// Version of Prometheus to be deployed.
 	Version string `json:"version,omitempty"`
 	// When a Prometheus deployment is paused, no actions except for deletion

pkg/client/monitoring/v1/zz_generated.deepcopy.go

@@ -451,6 +451,15 @@ func (in *PrometheusSpec) DeepCopyInto(out *PrometheusSpec) {
 			(*in).DeepCopyInto(*out)
 		}
 	}
+	if in.ServiceMonitorNamespaceSelector != nil {
+		in, out := &in.ServiceMonitorNamespaceSelector, &out.ServiceMonitorNamespaceSelector
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(meta_v1.LabelSelector)
+			(*in).DeepCopyInto(*out)
+		}
+	}
 	if in.ImagePullSecrets != nil {
 		in, out := &in.ImagePullSecrets, &out.ImagePullSecrets
 		*out = make([]core_v1.LocalObjectReference, len(*in))

pkg/prometheus/operator.go

@@ -62,6 +62,7 @@ type Operator struct {
 	cmapInf cache.SharedIndexInformer
 	secrInf cache.SharedIndexInformer
 	ssetInf cache.SharedIndexInformer
+	nsInf   cache.SharedIndexInformer
 
 	queue workqueue.RateLimitingInterface
 
@@ -239,6 +240,11 @@ func New(conf Config, logger log.Logger) (*Operator, error) {
 		UpdateFunc: c.handleUpdateStatefulSet,
 	})
 
+	c.nsInf = cache.NewSharedIndexInformer(
+		cache.NewListWatchFromClient(c.kclient.Core().RESTClient(), "namespaces", metav1.NamespaceAll, fields.Everything()),
+		&v1.Namespace{}, resyncPeriod, cache.Indexers{},
+	)
+
 	return c, nil
 }
 
@@ -291,6 +297,7 @@ func (c *Operator) Run(stopc <-chan struct{}) error {
 	go c.cmapInf.Run(stopc)
 	go c.secrInf.Run(stopc)
 	go c.ssetInf.Run(stopc)
+	go c.nsInf.Run(stopc)
 
 	if c.kubeletSyncEnabled {
 		go c.reconcileNodeEndpoints(stopc)
@@ -1037,22 +1044,37 @@ func (c *Operator) createConfig(p *monitoringv1.Prometheus, ruleFileConfigMaps [
 }
 
 func (c *Operator) selectServiceMonitors(p *monitoringv1.Prometheus) (map[string]*monitoringv1.ServiceMonitor, error) {
+	namespaces := []string{}
 	// Selectors might overlap. Deduplicate them along the keyFunc.
 	res := make(map[string]*monitoringv1.ServiceMonitor)
 
-	selector, err := metav1.LabelSelectorAsSelector(p.Spec.ServiceMonitorSelector)
+	servMonSelector, err := metav1.LabelSelectorAsSelector(p.Spec.ServiceMonitorSelector)
 	if err != nil {
 		return nil, err
 	}
 
-	// Only service monitors within the same namespace as the Prometheus
-	// object can belong to it.
-	cache.ListAllByNamespace(c.smonInf.GetIndexer(), p.Namespace, selector, func(obj interface{}) {
-		k, ok := c.keyFunc(obj)
-		if ok {
-			res[k] = obj.(*monitoringv1.ServiceMonitor)
+	// If 'ServiceMonitorNamespaceSelector' is empty, only check own namespace.
+	if p.Spec.ServiceMonitorNamespaceSelector == nil || p.Spec.ServiceMonitorNamespaceSelector.Size() == 0 {
+		namespaces = append(namespaces, p.Namespace)
+	} else {
+		servMonNSSelector, err := metav1.LabelSelectorAsSelector(p.Spec.ServiceMonitorNamespaceSelector)
+		if err != nil {
+			return nil, err
 		}
-	})
+
+		cache.ListAll(c.nsInf.GetStore(), servMonNSSelector, func(obj interface{}) {
+			namespaces = append(namespaces, obj.(*v1.Namespace).Name)
+		})
+	}
+
+	for _, ns := range namespaces {
+		cache.ListAllByNamespace(c.smonInf.GetIndexer(), ns, servMonSelector, func(obj interface{}) {
+			k, ok := c.keyFunc(obj)
+			if ok {
+				res[k] = obj.(*monitoringv1.ServiceMonitor)
+			}
+		})
+	}
 
 	return res, nil
 }

test/e2e/prometheus_test.go

@@ -21,6 +21,7 @@ import (
 	"log"
 	"reflect"
 	"sort"
+	"strings"
 	"testing"
 	"time"
 
@@ -496,6 +497,60 @@ func TestPrometheusDiscoverTargetPort(t *testing.T) {
 	}
 }
 
+func TestPromOpMatchPromAndServMonInDiffNSs(t *testing.T) {
+	ctx := framework.NewTestCtx(t)
+	defer ctx.Cleanup(t)
+	prometheusNSName := ctx.CreateNamespace(t, framework.KubeClient)
+	serviceMonitorNSName := ctx.CreateNamespace(t, framework.KubeClient)
+	ctx.SetupPrometheusRBAC(t, prometheusNSName, framework.KubeClient)
+
+	if err := testFramework.AddLabelsToNamespace(
+		framework.KubeClient,
+		serviceMonitorNSName,
+		map[string]string{"team": "frontend"},
+	); err != nil {
+		t.Fatal(err)
+	}
+
+	group := "sample-app"
+
+	prometheusJobName := serviceMonitorNSName + "/" + group
+
+	prometheusName := "test"
+	svc := framework.MakePrometheusService(prometheusName, group, v1.ServiceTypeClusterIP)
+
+	s := framework.MakeBasicServiceMonitor(group)
+
+	if _, err := framework.MonClient.ServiceMonitors(serviceMonitorNSName).Create(s); err != nil {
+		t.Fatal("Creating ServiceMonitor failed: ", err)
+	}
+
+	p := framework.MakeBasicPrometheus(prometheusNSName, prometheusName, group, 1)
+	p.Spec.ServiceMonitorNamespaceSelector = &metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"team": "frontend",
+		},
+	}
+	if err := framework.CreatePrometheusAndWaitUntilReady(prometheusNSName, p); err != nil {
+		t.Fatal(err)
+	}
+
+	if finalizerFn, err := testFramework.CreateServiceAndWaitUntilReady(framework.KubeClient, prometheusNSName, svc); err != nil {
+		t.Fatal(errors.Wrap(err, "creating prometheus service failed"))
+	} else {
+		ctx.AddFinalizerFn(finalizerFn)
+	}
+
+	resp, err := framework.QueryPrometheusSVC(prometheusNSName, svc.Name, "/api/v1/status/config", map[string]string{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if strings.Count(string(resp), prometheusJobName) != 1 {
+		t.Fatalf("expected Prometheus operator to configure Prometheus in ns '%v' to scrape the service monitor in ns '%v'", prometheusNSName, serviceMonitorNSName)
+	}
+}
+
 func isDiscoveryWorking(ns, svcName, prometheusName string) func() (bool, error) {
 	return func() (bool, error) {
 		pods, err := framework.KubeClient.CoreV1().Pods(ns).List(prometheus.ListOptions(prometheusName))

test/framework/namespace.go

@@ -57,3 +57,25 @@ func (ctx *TestCtx) CreateNamespace(t *testing.T, kubeClient kubernetes.Interfac
 func DeleteNamespace(kubeClient kubernetes.Interface, name string) error {
 	return kubeClient.Core().Namespaces().Delete(name, nil)
 }
+
+func AddLabelsToNamespace(kubeClient kubernetes.Interface, name string, additionalLabels map[string]string) error {
+	ns, err := kubeClient.CoreV1().Namespaces().Get(name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	if ns.Labels == nil {
+		ns.Labels = map[string]string{}
+	}
+
+	for k, v := range additionalLabels {
+		ns.Labels[k] = v
+	}
+
+	_, err = kubeClient.CoreV1().Namespaces().Update(ns)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}