prometheus: mount rule files based on label selector

Documentation/api.md

@@ -125,6 +125,7 @@ Specification of the desired behavior of the Prometheus cluster. More info: http
 | externalUrl | The external URL the Prometheus instances will be available under. This is necessary to generate correct URLs. This is necessary if Prometheus is not served from root of a DNS name. | string | false |
 | routePrefix | The route prefix Prometheus registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`. | string | false |
 | storage | Storage spec to specify how storage shall be used. | *[StorageSpec](#storagespec) | false |
+| ruleSelector | A selector to select which ConfigMaps to mount for loading rule files from. | *[metav1.LabelSelector](https://kubernetes.io/docs/api-reference/v1/definitions/#_unversioned_labelselector) | false |
 | alerting | Define details regarding alerting. | [AlertingSpec](#alertingspec) | false |
 | resources | Define resources requests and limits for single Pods. | [v1.ResourceRequirements](https://kubernetes.io/docs/api-reference/v1/definitions/#_v1_resourcerequirements) | false |
 | nodeSelector | Define which Nodes the Pods are scheduled on. | map[string]string | false |

Documentation/design.md

@@ -12,7 +12,7 @@ The third party resources that the Prometheus Operator introduces are:
 
 The `Prometheus` third party resource (TPR) declaratively defines a desired Prometheus setup to run in a Kubernetes cluster. It provides options to configure replication, persistent storage, and Alertmanagers to which the deployed Prometheus instances send alerts to.
 
-For each `Prometheus` TPR, the Operator deploys a properly configured `StatefulSet` in the same namespace. The Prometheus `Pod`s are configured to mount a `Secret` called `<prometheus-name>` containing the configuration for Prometheus and a `ConfigMap` called `<prometheus-name>-rules`, which holds Prometheus rule files that may contain alerting and recording rules.
+For each `Prometheus` TPR, the Operator deploys a properly configured `StatefulSet` in the same namespace. The Prometheus `Pod`s are configured to mount a `Secret` called `<prometheus-name>` containing the configuration for Prometheus.
 
 The TPR allows to specify which `ServiceMonitor`s should be covered by the deployed Prometheus instances based on label selection. The Operator then generates a configuration based on the included `ServiceMonitor`s and updates it in the `Secret` containing the configuration. It continuously does so for all changes that are made to `ServiceMonitor`s or the `Prometheus` TPR itself.
 

Documentation/user-guides/alerting.md

@@ -93,9 +93,15 @@ spec:
   resources:
     requests:
       memory: 400Mi
+  ruleSelector:
+    matchLabels:
+      role: prometheus-rulefiles
+      prometheus: example
 ```
 
-Prometheus rule files are held in a `ConfigMap` called `prometheus-<prometheus-object-name>-rules`. All top level files that end with the `.rules` extension will be loaded by a Prometheus instance.
+Prometheus rule files are held in a `ConfigMap`s. The `ConfigMap`s to mount rule files from are selected with a label selector field called `ruleSelector` in the Prometheus object, as seen above. All top level files that end with the `.rules` extension will be loaded.
+
+The best practice is to label the `ConfigMap`s containing rule files with `role: prometheus-rulefiles` as well as the name of the Prometheus object, `prometheus: example` in this case.
 
 [embedmd]:# (../../example/user-guides/alerting/prometheus-example-rules.yaml)
 ```yaml
@@ -103,14 +109,15 @@ kind: ConfigMap
 apiVersion: v1
 metadata:
   name: prometheus-example-rules
+  labels:
+    role: prometheus-rulefiles
+    prometheus: example
 data:
   example.rules: |
     ALERT ExampleAlert
     IF vector(1)
 ```
 
-> Note the Prometheus Operator will create an empty `ConfigMap` if it does not already exist.
-
 That example `ConfigMap` always immediately triggers an alert, which is only for demonstration purposes. To validate that everything is working properly have a look at each of the Prometheus web UIs.
 
 To be able to view the web UI without a `Service`, `kubectl`'s proxy functionality can be used.

example/user-guides/alerting/prometheus-example-rules.yaml

@@ -2,6 +2,9 @@ kind: ConfigMap
 apiVersion: v1
 metadata:
   name: prometheus-example-rules
+  labels:
+    role: prometheus-rulefiles
+    prometheus: example
 data:
   example.rules: |
     ALERT ExampleAlert

example/user-guides/alerting/prometheus-example.yaml

@@ -15,3 +15,7 @@ spec:
   resources:
     requests:
       memory: 400Mi
+  ruleSelector:
+    matchLabels:
+      role: prometheus-rulefiles
+      prometheus: example

pkg/client/monitoring/v1alpha1/types.go

@@ -73,6 +73,8 @@ type PrometheusSpec struct {
 	RoutePrefix string `json:"routePrefix,omitempty"`
 	// Storage spec to specify how storage shall be used.
 	Storage *StorageSpec `json:"storage,omitempty"`
+	// A selector to select which ConfigMaps to mount for loading rule files from.
+	RuleSelector *metav1.LabelSelector `json:"ruleSelector,omitempty"`
 	// Define details regarding alerting.
 	Alerting AlertingSpec `json:"alerting,omitempty"`
 	// Define resources requests and limits for single Pods.

pkg/prometheus/operator.go

@@ -154,7 +154,8 @@ func New(conf Config, logger log.Logger) (*Operator, error) {
 		&v1.ConfigMap{}, resyncPeriod, cache.Indexers{},
 	)
 	c.cmapInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
-		DeleteFunc: c.handleConfigmapDelete,
+		AddFunc:    c.handleConfigMapAdd,
+		DeleteFunc: c.handleConfigMapDelete,
 	})
 	c.secrInf = cache.NewSharedIndexInformer(
 		cache.NewListWatchFromClient(c.kclient.Core().RESTClient(), "secrets", api.NamespaceAll, nil),
@@ -392,24 +393,17 @@ func (c *Operator) handleSecretDelete(obj interface{}) {
 	}
 }
 
-func (c *Operator) handleConfigmapDelete(obj interface{}) {
+func (c *Operator) handleConfigMapAdd(obj interface{}) {
 	o, ok := c.getObject(obj)
-	if !ok {
-		return
+	if ok {
+		c.enqueueForNamespace(o.GetNamespace())
 	}
+}
 
-	key, ok := c.keyFunc(o)
-	if !ok {
-		return
-	}
-	key = strings.TrimSuffix(key, "-rules")
-
-	_, exists, err := c.promInf.GetIndexer().GetByKey(key)
-	if err != nil {
-		c.logger.Log("msg", "index lookup failed", "err", err)
-	}
-	if exists {
-		c.enqueue(key)
+func (c *Operator) handleConfigMapDelete(obj interface{}) {
+	o, ok := c.getObject(obj)
+	if ok {
+		c.enqueueForNamespace(o.GetNamespace())
 	}
 }
 
@@ -566,24 +560,25 @@ func (c *Operator) sync(key string) error {
 
 	c.logger.Log("msg", "sync prometheus", "key", key)
 
+	ruleFileConfigMaps, err := c.ruleFileConfigMaps(p)
+	if err != nil {
+		return errors.Wrap(err, "retrieving rule file configmaps failed")
+	}
+
 	// If no service monitor selectors are configured, the user wants to manage
 	// configuration himself.
 	if p.Spec.ServiceMonitorSelector != nil {
 		// We just always regenerate the configuration to be safe.
-		if err := c.createConfig(p); err != nil {
+		if err := c.createConfig(p, len(ruleFileConfigMaps)); err != nil {
 			return errors.Wrap(err, "creating config failed")
 		}
 	}
 
-	// Create Secret and ConfigMap if they don't exist.
+	// Create Secret if it doesn't exist.
 	sClient := c.kclient.Core().Secrets(p.Namespace)
 	if _, err := sClient.Create(makeEmptyConfig(p.Name)); err != nil && !apierrors.IsAlreadyExists(err) {
 		return errors.Wrap(err, "creating empty config file failed")
 	}
-	cmClient := c.kclient.Core().ConfigMaps(p.Namespace)
-	if _, err := cmClient.Create(makeEmptyRules(p.Name)); err != nil && !apierrors.IsAlreadyExists(err) {
-		return errors.Wrap(err, "creating empty rules file failed")
-	}
 
 	// Create governing service if it doesn't exist.
 	svcClient := c.kclient.Core().Services(p.Namespace)
@@ -599,12 +594,12 @@ func (c *Operator) sync(key string) error {
 	}
 
 	if !exists {
-		if _, err := ssetClient.Create(makeStatefulSet(*p, nil, &c.config)); err != nil {
+		if _, err := ssetClient.Create(makeStatefulSet(*p, nil, &c.config, ruleFileConfigMaps)); err != nil {
 			return errors.Wrap(err, "creating statefulset failed")
 		}
 		return nil
 	}
-	if _, err := ssetClient.Update(makeStatefulSet(*p, obj.(*v1beta1.StatefulSet), &c.config)); err != nil {
+	if _, err := ssetClient.Update(makeStatefulSet(*p, obj.(*v1beta1.StatefulSet), &c.config, ruleFileConfigMaps)); err != nil {
 		return errors.Wrap(err, "updating statefulset failed")
 	}
 
@@ -616,6 +611,24 @@ func (c *Operator) sync(key string) error {
 	return nil
 }
 
+func (c *Operator) ruleFileConfigMaps(p *v1alpha1.Prometheus) ([]*v1.ConfigMap, error) {
+	res := []*v1.ConfigMap{}
+
+	ruleSelector, err := metav1.LabelSelectorAsSelector(p.Spec.RuleSelector)
+	if err != nil {
+		return nil, err
+	}
+
+	cache.ListAllByNamespace(c.cmapInf.GetIndexer(), p.Namespace, ruleSelector, func(obj interface{}) {
+		_, ok := c.keyFunc(obj)
+		if ok {
+			res = append(res, obj.(*v1.ConfigMap))
+		}
+	})
+
+	return res, nil
+}
+
 func ListOptions(name string) metav1.ListOptions {
 	return metav1.ListOptions{
 		LabelSelector: fields.SelectorFromSet(fields.Set(map[string]string{
@@ -744,27 +757,28 @@ func (c *Operator) destroyPrometheus(key string) error {
 	if err := s.Delete(sset.Name, nil); err != nil {
 		return errors.Wrap(err, "deleting config file failed")
 	}
-	cm := c.kclient.Core().ConfigMaps(sset.Namespace)
-	if err := cm.Delete(fmt.Sprintf("%s-rules", sset.Name), nil); err != nil {
-		return errors.Wrap(err, "deleting rules file failed")
-	}
+
 	return nil
 }
 
-func (c *Operator) createConfig(p *v1alpha1.Prometheus) error {
+func (c *Operator) createConfig(p *v1alpha1.Prometheus, ruleFileConfigMaps int) error {
 	smons, err := c.selectServiceMonitors(p)
 	if err != nil {
 		return errors.Wrap(err, "selecting ServiceMonitors failed")
 	}
 	// Update secret based on the most recent configuration.
-	conf, err := generateConfig(p, smons)
+	conf, err := generateConfig(p, smons, ruleFileConfigMaps)
 	if err != nil {
 		return errors.Wrap(err, "generating config failed")
 	}
 
 	s := &v1.Secret{
 		ObjectMeta: apimetav1.ObjectMeta{
-			Name: configSecretName(p.Name),
+			Name:   configSecretName(p.Name),
+			Labels: managedByOperatorLabels,
+			Annotations: map[string]string{
+				"generated": "true",
+			},
 		},
 		Data: map[string][]byte{
 			"prometheus.yaml": []byte(conf),

pkg/prometheus/promcfg.go

@@ -33,7 +33,11 @@ func sanitizeLabelName(name string) string {
 	return invalidLabelCharRE.ReplaceAllString(name, "_")
 }
 
-func generateConfig(p *v1alpha1.Prometheus, mons map[string]*v1alpha1.ServiceMonitor) ([]byte, error) {
+func configMapRuleFileFolder(configMapNumber int) string {
+	return fmt.Sprintf("/etc/prometheus/rules/rules-%d/", configMapNumber)
+}
+
+func generateConfig(p *v1alpha1.Prometheus, mons map[string]*v1alpha1.ServiceMonitor, ruleConfigMaps int) ([]byte, error) {
 	cfg := map[string]interface{}{}
 
 	cfg["global"] = map[string]string{
@@ -41,7 +45,13 @@ func generateConfig(p *v1alpha1.Prometheus, mons map[string]*v1alpha1.ServiceMon
 		"scrape_interval":     "30s",
 	}
 
-	cfg["rule_files"] = []string{"/etc/prometheus/rules/*.rules"}
+	if ruleConfigMaps > 0 {
+		configMaps := make([]string, ruleConfigMaps)
+		for i := 0; i < ruleConfigMaps; i++ {
+			configMaps[i] = configMapRuleFileFolder(i) + "*.rules"
+		}
+		cfg["rule_files"] = configMaps
+	}
 
 	var scrapeConfigs []interface{}
 	for _, mon := range mons {

pkg/prometheus/promcfg_test.go

@@ -47,7 +47,7 @@ func TestConfigGenerationNonNamespacedAnnotation(t *testing.T) {
 		},
 	}
 
-	config, err := generateConfig(p, smons)
+	config, err := generateConfig(p, smons, 0)
 	if err != nil {
 		t.Fatal("Config generation failed: ", err)
 	}

pkg/prometheus/statefulset.go

@@ -36,10 +36,13 @@ const (
 )
 
 var (
-	minReplicas int32 = 1
+	minReplicas             int32 = 1
+	managedByOperatorLabels       = map[string]string{
+		"managed-by": "prometheus-operator",
+	}
 )
 
-func makeStatefulSet(p v1alpha1.Prometheus, old *v1beta1.StatefulSet, config *Config) *v1beta1.StatefulSet {
+func makeStatefulSet(p v1alpha1.Prometheus, old *v1beta1.StatefulSet, config *Config, ruleConfigMaps []*v1.ConfigMap) *v1beta1.StatefulSet {
 	// TODO(fabxc): is this the right point to inject defaults?
 	// Ideally we would do it before storing but that's currently not possible.
 	// Potentially an update handler on first insertion.
@@ -70,7 +73,7 @@ func makeStatefulSet(p v1alpha1.Prometheus, old *v1beta1.StatefulSet, config *Co
 			Labels:      p.ObjectMeta.Labels,
 			Annotations: p.ObjectMeta.Annotations,
 		},
-		Spec: makeStatefulSetSpec(p, config),
+		Spec: makeStatefulSetSpec(p, config, ruleConfigMaps),
 	}
 	if vc := p.Spec.Storage; vc == nil {
 		statefulset.Spec.Template.Spec.Volumes = append(statefulset.Spec.Template.Spec.Volumes, v1.Volume{
@@ -104,10 +107,44 @@ func makeStatefulSet(p v1alpha1.Prometheus, old *v1beta1.StatefulSet, config *Co
 	return statefulset
 }
 
+func volumesInfoFromRuleConfigMaps(ruleConfigMaps []*v1.ConfigMap) ([]v1.Volume, []v1.VolumeMount, []string) {
+	volumes := make([]v1.Volume, len(ruleConfigMaps))
+	volumeMounts := make([]v1.VolumeMount, len(ruleConfigMaps))
+	ruleFileVolumeArgs := make([]string, len(ruleConfigMaps))
+
+	for i, c := range ruleConfigMaps {
+		volumeName := fmt.Sprintf("rules-%d", i)
+		ruleFilePath := configMapRuleFileFolder(i)
+
+		volumes[i] = v1.Volume{
+			Name: volumeName,
+			VolumeSource: v1.VolumeSource{
+				ConfigMap: &v1.ConfigMapVolumeSource{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: c.Name,
+					},
+				},
+			},
+		}
+		volumeMounts[i] = v1.VolumeMount{
+			Name:      volumeName,
+			ReadOnly:  true,
+			MountPath: ruleFilePath,
+		}
+		ruleFileVolumeArgs[i] = fmt.Sprintf("-volume-dir=%s", ruleFilePath)
+	}
+
+	return volumes, volumeMounts, ruleFileVolumeArgs
+}
+
 func makeEmptyConfig(name string) *v1.Secret {
 	return &v1.Secret{
 		ObjectMeta: apimetav1.ObjectMeta{
-			Name: configSecretName(name),
+			Name:   configSecretName(name),
+			Labels: managedByOperatorLabels,
+			Annotations: map[string]string{
+				"empty": "true",
+			},
 		},
 		Data: map[string][]byte{
 			"prometheus.yaml": []byte{},
@@ -115,14 +152,6 @@ func makeEmptyConfig(name string) *v1.Secret {
 	}
 }
 
-func makeEmptyRules(name string) *v1.ConfigMap {
-	return &v1.ConfigMap{
-		ObjectMeta: apimetav1.ObjectMeta{
-			Name: rulesConfigMapName(name),
-		},
-	}
-}
-
 func makeStatefulSetService(p *v1alpha1.Prometheus) *v1.Service {
 	svc := &v1.Service{
 		ObjectMeta: apimetav1.ObjectMeta{
@@ -148,7 +177,7 @@ func makeStatefulSetService(p *v1alpha1.Prometheus) *v1.Service {
 	return svc
 }
 
-func makeStatefulSetSpec(p v1alpha1.Prometheus, c *Config) v1beta1.StatefulSetSpec {
+func makeStatefulSetSpec(p v1alpha1.Prometheus, c *Config, ruleConfigMaps []*v1.ConfigMap) v1beta1.StatefulSetSpec {
 	// Prometheus may take quite long to shut down to checkpoint existing data.
 	// Allow up to 10 minutes for clean termination.
 	terminationGracePeriod := int64(600)
@@ -195,6 +224,45 @@ func makeStatefulSetSpec(p v1alpha1.Prometheus, c *Config) v1beta1.StatefulSetSp
 		Path:   path.Clean(webRoutePrefix + "/-/reload"),
 	}
 
+	ruleFileVolumes, ruleFileVolumeMounts, ruleFileVolumeArgs := volumesInfoFromRuleConfigMaps(ruleConfigMaps)
+
+	volumes := append([]v1.Volume{
+		{
+			Name: "config",
+			VolumeSource: v1.VolumeSource{
+				Secret: &v1.SecretVolumeSource{
+					SecretName: configSecretName(p.Name),
+				},
+			},
+		},
+	}, ruleFileVolumes...)
+
+	promVolumeMounts := append([]v1.VolumeMount{
+		{
+			Name:      "config",
+			ReadOnly:  true,
+			MountPath: "/etc/prometheus/config",
+		},
+		{
+			Name:      volumeName(p.Name),
+			MountPath: "/var/prometheus/data",
+			SubPath:   subPathForStorage(p.Spec.Storage),
+		},
+	}, ruleFileVolumeMounts...)
+
+	configReloadVolumeMounts := append([]v1.VolumeMount{
+		{
+			Name:      "config",
+			ReadOnly:  true,
+			MountPath: "/etc/prometheus/config",
+		},
+	}, ruleFileVolumeMounts...)
+
+	configReloadArgs := append([]string{
+		fmt.Sprintf("-webhook-url=%s", localReloadURL),
+		"-volume-dir=/etc/prometheus/config",
+	}, ruleFileVolumeArgs...)
+
 	return v1beta1.StatefulSetSpec{
 		ServiceName: governingServiceName,
 		Replicas:    p.Spec.Replicas,
@@ -217,24 +285,8 @@ func makeStatefulSetSpec(p v1alpha1.Prometheus, c *Config) v1beta1.StatefulSetSp
 								Protocol:      v1.ProtocolTCP,
 							},
 						},
-						Args: promArgs,
-						VolumeMounts: []v1.VolumeMount{
-							{
-								Name:      "config",
-								ReadOnly:  true,
-								MountPath: "/etc/prometheus/config",
-							},
-							{
-								Name:      "rules",
-								ReadOnly:  true,
-								MountPath: "/etc/prometheus/rules",
-							},
-							{
-								Name:      volumeName(p.Name),
-								MountPath: "/var/prometheus/data",
-								SubPath:   subPathForStorage(p.Spec.Storage),
-							},
-						},
+						Args:         promArgs,
+						VolumeMounts: promVolumeMounts,
 						ReadinessProbe: &v1.Probe{
 							Handler: v1.Handler{
 								HTTPGet: &v1.HTTPGetAction{
@@ -251,25 +303,10 @@ func makeStatefulSetSpec(p v1alpha1.Prometheus, c *Config) v1beta1.StatefulSetSp
 						},
 						Resources: p.Spec.Resources,
 					}, {
-						Name:  "config-reloader",
-						Image: c.ConfigReloaderImage,
-						Args: []string{
-							fmt.Sprintf("-webhook-url=%s", localReloadURL),
-							"-volume-dir=/etc/prometheus/config",
-							"-volume-dir=/etc/prometheus/rules/",
-						},
-						VolumeMounts: []v1.VolumeMount{
-							{
-								Name:      "config",
-								ReadOnly:  true,
-								MountPath: "/etc/prometheus/config",
-							},
-							{
-								Name:      "rules",
-								ReadOnly:  true,
-								MountPath: "/etc/prometheus/rules",
-							},
-						},
+						Name:         "config-reloader",
+						Image:        c.ConfigReloaderImage,
+						Args:         configReloadArgs,
+						VolumeMounts: configReloadVolumeMounts,
 						Resources: v1.ResourceRequirements{
 							Limits: v1.ResourceList{
 								v1.ResourceCPU:    resource.MustParse("5m"),
@@ -281,26 +318,7 @@ func makeStatefulSetSpec(p v1alpha1.Prometheus, c *Config) v1beta1.StatefulSetSp
 				ServiceAccountName:            p.Spec.ServiceAccountName,
 				NodeSelector:                  p.Spec.NodeSelector,
 				TerminationGracePeriodSeconds: &terminationGracePeriod,
-				Volumes: []v1.Volume{
-					{
-						Name: "config",
-						VolumeSource: v1.VolumeSource{
-							Secret: &v1.SecretVolumeSource{
-								SecretName: configSecretName(p.Name),
-							},
-						},
-					},
-					{
-						Name: "rules",
-						VolumeSource: v1.VolumeSource{
-							ConfigMap: &v1.ConfigMapVolumeSource{
-								LocalObjectReference: v1.LocalObjectReference{
-									Name: rulesConfigMapName(p.Name),
-								},
-							},
-						},
-					},
-				},
+				Volumes: volumes,
 			},
 		},
 	}
@@ -310,10 +328,6 @@ func configSecretName(name string) string {
 	return prefixedName(name)
 }
 
-func rulesConfigMapName(name string) string {
-	return fmt.Sprintf("%s-rules", prefixedName(name))
-}
-
 func volumeName(name string) string {
 	return fmt.Sprintf("%s-db", prefixedName(name))
 }

pkg/prometheus/statefulset_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/pkg/api/v1"
 
 	"github.com/coreos/prometheus-operator/pkg/client/monitoring/v1alpha1"
 )
@@ -42,7 +43,7 @@ func TestStatefulSetLabelingAndAnnotations(t *testing.T) {
 			Labels:      labels,
 			Annotations: annotations,
 		},
-	}, nil, defaultTestConfig)
+	}, nil, defaultTestConfig, []*v1.ConfigMap{})
 
 	if !reflect.DeepEqual(labels, sset.Labels) || !reflect.DeepEqual(annotations, sset.Annotations) {
 		t.Fatal("Labels or Annotations are not properly being propagated to the StatefulSet")

test/e2e/framework/prometheus.go

@@ -42,6 +42,11 @@ func (f *Framework) MakeBasicPrometheus(name, group string, replicas int32) *v1a
 					"group": group,
 				},
 			},
+			RuleSelector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"role": "rulefile",
+				},
+			},
 			Resources: v1.ResourceRequirements{
 				Requests: v1.ResourceList{
 					v1.ResourceMemory: resource.MustParse("400Mi"),

test/e2e/prometheus_e2e_test.go

@@ -193,24 +193,40 @@ scrape_configs:
 func TestPrometheusReloadRules(t *testing.T) {
 	name := "test"
 
+	ruleFileConfigMap := &v1.ConfigMap{
+		ObjectMeta: apimetav1.ObjectMeta{
+			Name: fmt.Sprintf("prometheus-%s-rules", name),
+			Labels: map[string]string{
+				"role": "rulefile",
+			},
+		},
+		Data: map[string]string{
+			"test.rules": "",
+		},
+	}
+
+	_, err := framework.KubeClient.CoreV1().ConfigMaps(framework.Namespace.Name).Create(ruleFileConfigMap)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	defer func() {
 		if err := framework.DeletePrometheusAndWaitUntilGone(name); err != nil {
 			t.Fatal(err)
 		}
+
+		err = framework.KubeClient.CoreV1().ConfigMaps(framework.Namespace.Name).Delete(ruleFileConfigMap.Name, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
 	}()
 
 	if err := framework.CreatePrometheusAndWaitUntilReady(framework.MakeBasicPrometheus(name, name, 1)); err != nil {
 		t.Fatal(err)
 	}
 
-	_, err := framework.KubeClient.CoreV1().ConfigMaps(framework.Namespace.Name).Update(&v1.ConfigMap{
-		ObjectMeta: apimetav1.ObjectMeta{
-			Name: fmt.Sprintf("prometheus-%s-rules", name),
-		},
-		Data: map[string]string{
-			"test.rules": "",
-		},
-	})
+	ruleFileConfigMap.Data["test.rules"] = "# comment to trigger a configmap reload"
+	_, err = framework.KubeClient.CoreV1().ConfigMaps(framework.Namespace.Name).Update(ruleFileConfigMap)
 	if err != nil {
 		t.Fatal(err)
 	}