refactor: decouple pod labels from selector labels (#3294)

* refactor: decouple pod labels from selector labels

prometheus pods can not be rolled out without downtime when label's are changed

Fixes #3120

* chore: run go fmt

* fix unit tests

pkg/prometheus/statefulset.go

@@ -657,6 +657,10 @@ func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config, ruleConfigMapName
 
 	podAnnotations := map[string]string{}
 	podLabels := map[string]string{}
+	podSelectorLabels := map[string]string{
+		"app":        "prometheus",
+		"prometheus": p.Name,
+	}
 	if p.Spec.PodMetadata != nil {
 		if p.Spec.PodMetadata.Labels != nil {
 			for k, v := range p.Spec.PodMetadata.Labels {
@@ -670,9 +674,11 @@ func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config, ruleConfigMapName
 		}
 	}
 
-	podLabels["app"] = "prometheus"
-	podLabels["prometheus"] = p.Name
+	for k, v := range podSelectorLabels {
+		podLabels[k] = v
+	}
 
+	finalSelectorLabels := c.Labels.Merge(podSelectorLabels)
 	finalLabels := c.Labels.Merge(podLabels)
 
 	var additionalContainers []v1.Container
@@ -904,7 +910,7 @@ func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config, ruleConfigMapName
 			Type: appsv1.RollingUpdateStatefulSetStrategyType,
 		},
 		Selector: &metav1.LabelSelector{
-			MatchLabels: finalLabels,
+			MatchLabels: finalSelectorLabels,
 		},
 		Template: v1.PodTemplateSpec{
 			ObjectMeta: metav1.ObjectMeta{

pkg/prometheus/statefulset_test.go

@@ -101,6 +101,25 @@ func TestPodLabelsAnnotations(t *testing.T) {
 		t.Fatal("Pod annotaitons are not properly propagated")
 	}
 }
+func TestPodLabelsShouldNotBeSelectorLabels(t *testing.T) {
+	labels := map[string]string{
+		"testlabel": "testvalue",
+	}
+	sset, err := makeStatefulSet(monitoringv1.Prometheus{
+		ObjectMeta: metav1.ObjectMeta{},
+		Spec: monitoringv1.PrometheusSpec{
+			PodMetadata: &monitoringv1.EmbeddedObjectMetadata{
+				Labels: labels,
+			},
+		},
+	}, defaultTestConfig, nil, "")
+
+	require.NoError(t, err)
+
+	if sset.Spec.Selector.MatchLabels["testlabel"] == "testvalue" {
+		t.Fatal("Pod Selector are not properly propagated")
+	}
+}
 
 func TestStatefulSetPVC(t *testing.T) {
 	labels := map[string]string{