[Feature] Ensure ServiceMonitor labels (#605)

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Change Log
 
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
+- Add Labels and Annotations to ServiceMonitor
 
 ## [1.0.4](https://github.com/arangodb/kube-arangodb/tree/1.0.4) (2020-07-28)
 - Add Encryption Key rotation feature for ArangoDB EE 3.7+

chart/kube-arangodb/templates/deployment-operator/role.yaml

@@ -30,7 +30,7 @@ rules:
       verbs: ["get", "list", "watch"]
     - apiGroups: ["monitoring.coreos.com"]
       resources: ["servicemonitors"]
-      verbs: ["get", "create", "delete", "update"]
+      verbs: ["get", "create", "delete", "update", "list", "watch", "patch"]
 
 {{- end }}
 {{- end }}

dashboard/assets.go

@@ -1,3 +1,23 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
 package dashboard
 
 import (

main.go

@@ -279,6 +279,11 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper
 		return operator.Config{}, operator.Dependencies{}, maskAny(err)
 	}
 
+	kubeMonCli, err := k8sutil.NewKubeMonitoringV1Client()
+	if err != nil {
+		return operator.Config{}, operator.Dependencies{}, maskAny(err)
+	}
+
 	image, serviceAccount, err := getMyPodInfo(kubecli, namespace, name)
 	if err != nil {
 		return operator.Config{}, operator.Dependencies{}, maskAny(fmt.Errorf("Failed to get my pod's service account: %s", err))
@@ -320,6 +325,7 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper
 		LogService:                 logService,
 		KubeCli:                    kubecli,
 		KubeExtCli:                 kubeExtCli,
+		KubeMonitoringCli:          kubeMonCli,
 		CRCli:                      crCli,
 		EventRecorder:              eventRecorder,
 		LivenessProbe:              &livenessProbe,

pkg/deployment/context_impl.go

@@ -31,6 +31,8 @@ import (
 	"strconv"
 	"time"
 
+	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+
 	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
 
 	"github.com/arangodb/go-driver/http"
@@ -78,6 +80,10 @@ func (d *Deployment) GetKubeCli() kubernetes.Interface {
 	return d.deps.KubeCli
 }
 
+func (d *Deployment) GetMonitoringV1Cli() monitoringClient.MonitoringV1Interface {
+	return d.deps.KubeMonitoringCli
+}
+
 // GetLifecycleImage returns the image name containing the lifecycle helper (== name of operator image)
 func (d *Deployment) GetLifecycleImage() string {
 	return d.config.LifecycleImage

pkg/deployment/deployment.go

@@ -29,6 +29,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+
 	"github.com/arangodb/kube-arangodb/pkg/util/arangod/conn"
 
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
@@ -65,11 +67,12 @@ type Config struct {
 
 // Dependencies holds dependent services for a Deployment
 type Dependencies struct {
-	Log           zerolog.Logger
-	KubeCli       kubernetes.Interface
-	KubeExtCli    apiextensionsclient.Interface
-	DatabaseCRCli versioned.Interface
-	EventRecorder record.EventRecorder
+	Log               zerolog.Logger
+	KubeCli           kubernetes.Interface
+	KubeExtCli        apiextensionsclient.Interface
+	KubeMonitoringCli monitoringClient.MonitoringV1Interface
+	DatabaseCRCli     versioned.Interface
+	EventRecorder     record.EventRecorder
 }
 
 // deploymentEventType strongly typed type of event
@@ -237,7 +240,7 @@ func (d *Deployment) run() {
 	for {
 		select {
 		case <-d.stopCh:
-			cachedStatus, err := inspector.NewInspector(d.GetKubeCli(), d.GetNamespace())
+			cachedStatus, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetNamespace())
 			if err != nil {
 				log.Error().Err(err).Msg("Unable to get resources")
 			}

pkg/deployment/deployment_inspector.go

@@ -65,7 +65,7 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval
 	deploymentName := d.apiObject.GetName()
 	defer metrics.SetDuration(inspectDeploymentDurationGauges.WithLabelValues(deploymentName), start)
 
-	cachedStatus, err := inspector.NewInspector(d.GetKubeCli(), d.GetNamespace())
+	cachedStatus, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetNamespace())
 	if err != nil {
 		log.Error().Err(err).Msg("Unable to get resources")
 		return minInspectionInterval // Retry ASAP

pkg/deployment/deployment_run_test.go

@@ -62,7 +62,7 @@ func runTestCase(t *testing.T, testCase testCaseStruct) {
 
 		errs := 0
 		for {
-			cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetNamespace())
+			cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetNamespace())
 			require.NoError(t, err)
 			err = d.resources.EnsureSecrets(log.Logger, cache)
 			if err == nil {
@@ -104,7 +104,7 @@ func runTestCase(t *testing.T, testCase testCaseStruct) {
 		}
 
 		// Act
-		cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetNamespace())
+		cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetNamespace())
 		require.NoError(t, err)
 		err = d.resources.EnsurePods(cache)
 

pkg/deployment/deployment_suite_test.go

@@ -29,6 +29,8 @@ import (
 	"path/filepath"
 	"testing"
 
+	monitoringFakeClient "github.com/coreos/prometheus-operator/pkg/client/versioned/fake"
+
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes"
 
 	"github.com/arangodb/kube-arangodb/pkg/util/arangod/conn"
@@ -419,6 +421,7 @@ func createTestDeployment(config Config, arangoDeployment *api.ArangoDeployment)
 
 	eventRecorder := recordfake.NewFakeRecorder(10)
 	kubernetesClientSet := fake.NewSimpleClientset()
+	monitoringClientSet := monitoringFakeClient.NewSimpleClientset()
 
 	arangoDeployment.ObjectMeta = metav1.ObjectMeta{
 		Name:      testDeploymentName,
@@ -426,10 +429,11 @@ func createTestDeployment(config Config, arangoDeployment *api.ArangoDeployment)
 	}
 
 	deps := Dependencies{
-		Log:           zerolog.New(ioutil.Discard),
-		KubeCli:       kubernetesClientSet,
-		DatabaseCRCli: arangofake.NewSimpleClientset(&api.ArangoDeployment{}),
-		EventRecorder: eventRecorder,
+		Log:               zerolog.New(ioutil.Discard),
+		KubeCli:           kubernetesClientSet,
+		KubeMonitoringCli: monitoringClientSet.MonitoringV1(),
+		DatabaseCRCli:     arangofake.NewSimpleClientset(&api.ArangoDeployment{}),
+		EventRecorder:     eventRecorder,
 	}
 
 	d := &Deployment{

pkg/deployment/reconcile/plan_builder_test.go

@@ -28,6 +28,8 @@ import (
 	"io/ioutil"
 	"testing"
 
+	monitoring "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
+
 	"github.com/pkg/errors"
 
 	policy "k8s.io/api/policy/v1beta1"
@@ -548,6 +550,7 @@ func TestCreatePlan(t *testing.T) {
 		PVCS            map[string]*core.PersistentVolumeClaim
 		ServiceAccounts map[string]*core.ServiceAccount
 		PDBS            map[string]*policy.PodDisruptionBudget
+		ServiceMonitors map[string]*monitoring.ServiceMonitor
 	}{
 		{
 			Name: "Can not create plan for single deployment",
@@ -807,7 +810,7 @@ func TestCreatePlan(t *testing.T) {
 			if testCase.Helper != nil {
 				testCase.Helper(testCase.context.ArangoDeployment)
 			}
-			err, _ := r.CreatePlan(ctx, inspector.NewInspectorFromData(testCase.Pods, testCase.Secrets, testCase.PVCS, testCase.Services, testCase.ServiceAccounts, testCase.PDBS))
+			err, _ := r.CreatePlan(ctx, inspector.NewInspectorFromData(testCase.Pods, testCase.Secrets, testCase.PVCS, testCase.Services, testCase.ServiceAccounts, testCase.PDBS, testCase.ServiceMonitors))
 
 			// Assert
 			if testCase.ExpectedEvent != nil {

pkg/deployment/resources/annotations.go

@@ -25,6 +25,9 @@ package resources
 import (
 	"time"
 
+	monitoring "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
+	monitoringTypedClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
 
 	"github.com/arangodb/kube-arangodb/pkg/apis/deployment"
@@ -41,6 +44,7 @@ import (
 
 func (r *Resources) EnsureAnnotations(cachedStatus inspector.Inspector) error {
 	kubecli := r.context.GetKubeCli()
+	monitoringcli := r.context.GetMonitoringV1Cli()
 
 	log.Info().Msgf("Ensuring annotations")
 
@@ -99,6 +103,15 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspector.Inspector) error {
 		return err
 	}
 
+	if err := ensureServiceMonitorsAnnotations(monitoringcli.ServiceMonitors(r.context.GetNamespace()),
+		cachedStatus,
+		deployment.ArangoDeploymentResourceKind,
+		r.context.GetAPIObject().GetName(),
+		r.context.GetAPIObject().GetNamespace(),
+		r.context.GetSpec().Annotations); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -287,6 +300,43 @@ func setPvcAnnotations(client typedCore.PersistentVolumeClaimInterface, persiste
 	})
 }
 
+func ensureServiceMonitorsAnnotations(client monitoringTypedClient.ServiceMonitorInterface, cachedStatus inspector.Inspector, kind, name, namespace string, annotations map[string]string) error {
+	if err := cachedStatus.IterateServiceMonitors(func(serviceMonitor *monitoring.ServiceMonitor) error {
+		if !k8sutil.CompareAnnotations(serviceMonitor.GetAnnotations(), annotations) {
+			log.Info().Msgf("Replacing annotations for ServiceMonitors %s", serviceMonitor.Name)
+			if err := setServiceMonitorAnnotations(client, serviceMonitor, annotations); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}, func(serviceMonitor *monitoring.ServiceMonitor) bool {
+		return k8sutil.IsChildResource(kind, name, namespace, serviceMonitor)
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func setServiceMonitorAnnotations(client monitoringTypedClient.ServiceMonitorInterface, serviceMonitor *monitoring.ServiceMonitor, annotations map[string]string) error {
+	return utils.Retry(5, 200*time.Millisecond, func() error {
+		currentServiceMonitor, err := client.Get(serviceMonitor.Name, meta.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		currentServiceMonitor.Annotations = k8sutil.MergeAnnotations(k8sutil.GetSecuredAnnotations(currentServiceMonitor.Annotations), annotations)
+
+		_, err = client.Update(currentServiceMonitor)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
 func getObjectGroup(obj meta.Object) api.ServerGroup {
 	l := obj.GetLabels()
 	if len(l) == 0 {

pkg/deployment/resources/context.go

@@ -25,6 +25,8 @@ package resources
 import (
 	"context"
 
+	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+
 	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
 
 	driver "github.com/arangodb/go-driver"
@@ -61,6 +63,8 @@ type Context interface {
 	UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error
 	// GetKubeCli returns the kubernetes client
 	GetKubeCli() kubernetes.Interface
+	// GetMonitoringV1Cli returns monitoring client
+	GetMonitoringV1Cli() monitoringClient.MonitoringV1Interface
 	// GetLifecycleImage returns the image name containing the lifecycle helper (== name of operator image)
 	GetLifecycleImage() string
 	// GetOperatorUUIDImage returns the image name containing the uuid helper (== name of operator image)

pkg/deployment/resources/inspector/inspector.go

@@ -25,12 +25,15 @@ package inspector
 import (
 	"sync"
 
+	monitoring "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
+	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+
 	core "k8s.io/api/core/v1"
 	policy "k8s.io/api/policy/v1beta1"
 	"k8s.io/client-go/kubernetes"
 )
 
-func NewInspector(k kubernetes.Interface, namespace string) (Inspector, error) {
+func NewInspector(k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, namespace string) (Inspector, error) {
 	pods, err := podsToMap(k, namespace)
 	if err != nil {
 		return nil, err
@@ -61,11 +64,16 @@ func NewInspector(k kubernetes.Interface, namespace string) (Inspector, error) {
 		return nil, err
 	}
 
-	return NewInspectorFromData(pods, secrets, pvcs, services, serviceAccounts, podDisruptionBudgets), nil
+	serviceMonitors, err := serviceMonitorsToMap(m, namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewInspectorFromData(pods, secrets, pvcs, services, serviceAccounts, podDisruptionBudgets, serviceMonitors), nil
 }
 
 func NewEmptyInspector() Inspector {
-	return NewInspectorFromData(nil, nil, nil, nil, nil, nil)
+	return NewInspectorFromData(nil, nil, nil, nil, nil, nil, nil)
 }
 
 func NewInspectorFromData(pods map[string]*core.Pod,
@@ -73,7 +81,8 @@ func NewInspectorFromData(pods map[string]*core.Pod,
 	pvcs map[string]*core.PersistentVolumeClaim,
 	services map[string]*core.Service,
 	serviceAccounts map[string]*core.ServiceAccount,
-	podDisruptionBudgets map[string]*policy.PodDisruptionBudget) Inspector {
+	podDisruptionBudgets map[string]*policy.PodDisruptionBudget,
+	serviceMonitors map[string]*monitoring.ServiceMonitor) Inspector {
 	return &inspector{
 		pods:                 pods,
 		secrets:              secrets,
@@ -81,11 +90,12 @@ func NewInspectorFromData(pods map[string]*core.Pod,
 		services:             services,
 		serviceAccounts:      serviceAccounts,
 		podDisruptionBudgets: podDisruptionBudgets,
+		serviceMonitors:      serviceMonitors,
 	}
 }
 
 type Inspector interface {
-	Refresh(k kubernetes.Interface, namespace string) error
+	Refresh(k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, namespace string) error
 
 	Pod(name string) (*core.Pod, bool)
 	IteratePods(action PodAction, filters ...PodFilter) error
@@ -104,6 +114,9 @@ type Inspector interface {
 
 	PodDisruptionBudget(name string) (*policy.PodDisruptionBudget, bool)
 	IteratePodDisruptionBudgets(action PodDisruptionBudgetAction, filters ...PodDisruptionBudgetFilter) error
+
+	ServiceMonitor(name string) (*monitoring.ServiceMonitor, bool)
+	IterateServiceMonitors(action ServiceMonitorAction, filters ...ServiceMonitorFilter) error
 }
 
 type inspector struct {
@@ -115,12 +128,14 @@ type inspector struct {
 	services             map[string]*core.Service
 	serviceAccounts      map[string]*core.ServiceAccount
 	podDisruptionBudgets map[string]*policy.PodDisruptionBudget
+	serviceMonitors      map[string]*monitoring.ServiceMonitor
 
 	ns string
 	k  kubernetes.Interface
+	m  monitoringClient.MonitoringV1Interface
 }
 
-func (i *inspector) Refresh(k kubernetes.Interface, namespace string) error {
+func (i *inspector) Refresh(k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, namespace string) error {
 	i.lock.Lock()
 	defer i.lock.Unlock()
 
@@ -154,12 +169,18 @@ func (i *inspector) Refresh(k kubernetes.Interface, namespace string) error {
 		return err
 	}
 
+	serviceMonitors, err := serviceMonitorsToMap(m, namespace)
+	if err != nil {
+		return err
+	}
+
 	i.pods = pods
 	i.secrets = secrets
 	i.pvcs = pvcs
 	i.services = services
 	i.serviceAccounts = serviceAccounts
 	i.podDisruptionBudgets = podDisruptionBudgets
+	i.serviceMonitors = serviceMonitors
 
 	return nil
 }

pkg/deployment/resources/inspector/sms.go

@@ -0,0 +1,117 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package inspector
+
+import (
+	monitoring "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
+	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+	"github.com/pkg/errors"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type ServiceMonitorFilter func(serviceMonitor *monitoring.ServiceMonitor) bool
+type ServiceMonitorAction func(serviceMonitor *monitoring.ServiceMonitor) error
+
+func (i *inspector) IterateServiceMonitors(action ServiceMonitorAction, filters ...ServiceMonitorFilter) error {
+	i.lock.Lock()
+	defer i.lock.Unlock()
+
+	for _, serviceMonitor := range i.serviceMonitors {
+		if err := i.iterateServiceMonitor(serviceMonitor, action, filters...); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (i *inspector) iterateServiceMonitor(serviceMonitor *monitoring.ServiceMonitor, action ServiceMonitorAction, filters ...ServiceMonitorFilter) error {
+	for _, filter := range filters {
+		if !filter(serviceMonitor) {
+			return nil
+		}
+	}
+
+	return action(serviceMonitor)
+}
+
+func (i *inspector) ServiceMonitor(name string) (*monitoring.ServiceMonitor, bool) {
+	i.lock.Lock()
+	defer i.lock.Unlock()
+
+	serviceMonitor, ok := i.serviceMonitors[name]
+	if !ok {
+		return nil, false
+	}
+
+	return serviceMonitor, true
+}
+
+func serviceMonitorsToMap(m monitoringClient.MonitoringV1Interface, namespace string) (map[string]*monitoring.ServiceMonitor, error) {
+	serviceMonitors, err := getServiceMonitors(m, namespace, "")
+	if err != nil {
+		return nil, err
+	}
+
+	serviceMonitorMap := map[string]*monitoring.ServiceMonitor{}
+
+	for _, serviceMonitor := range serviceMonitors {
+		_, exists := serviceMonitorMap[serviceMonitor.GetName()]
+		if exists {
+			return nil, errors.Errorf("ServiceMonitor %s already exists in map, error received", serviceMonitor.GetName())
+		}
+
+		serviceMonitorMap[serviceMonitor.GetName()] = serviceMonitor
+	}
+
+	return serviceMonitorMap, nil
+}
+
+func getServiceMonitors(m monitoringClient.MonitoringV1Interface, namespace, cont string) ([]*monitoring.ServiceMonitor, error) {
+	serviceMonitors, err := m.ServiceMonitors(namespace).List(meta.ListOptions{
+		Limit:    128,
+		Continue: cont,
+	})
+
+	if err != nil {
+		return []*monitoring.ServiceMonitor{}, nil
+	}
+
+	return serviceMonitors.Items, nil
+}
+
+func FilterServiceMonitorsByLabels(labels map[string]string) ServiceMonitorFilter {
+	return func(serviceMonitor *monitoring.ServiceMonitor) bool {
+		for key, value := range labels {
+			v, ok := serviceMonitor.Labels[key]
+			if !ok {
+				return false
+			}
+
+			if v != value {
+				return false
+			}
+		}
+
+		return true
+	}
+}

pkg/deployment/resources/labels.go

@@ -25,6 +25,8 @@ package resources
 import (
 	"encoding/json"
 
+	monitoring "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
+
 	"github.com/arangodb/kube-arangodb/pkg/deployment/patch"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
@@ -60,6 +62,10 @@ func (r *Resources) EnsureLabels(cachedStatus inspector.Inspector) error {
 		return err
 	}
 
+	if err := r.EnsureServiceMonitorsLabels(cachedStatus); err != nil {
+		return err
+	}
+
 	if err := r.EnsurePodsLabels(cachedStatus); err != nil {
 		return err
 	}
@@ -171,6 +177,38 @@ func (r *Resources) EnsureServicesLabels(cachedStatus inspector.Inspector) error
 	return nil
 }
 
+func (r *Resources) EnsureServiceMonitorsLabels(cachedStatus inspector.Inspector) error {
+	changed := false
+	if err := cachedStatus.IterateServiceMonitors(func(serviceMonitor *monitoring.ServiceMonitor) error {
+		if p := ensureLabelsFromMaps(serviceMonitor, r.context.GetSpec().Labels, r.context.GetSpec().GetServerGroupSpec(getObjectGroup(serviceMonitor)).Labels); len(p) != 0 {
+			patch, err := json.Marshal(p)
+			if err != nil {
+				return err
+			}
+			r.log.Info().Int("changes", len(p)).Msgf("Updating labels for ServiceMonitor %s", serviceMonitor.GetName())
+
+			if _, err = r.context.GetMonitoringV1Cli().ServiceMonitors(r.context.GetAPIObject().GetNamespace()).Patch(serviceMonitor.GetName(), types.JSONPatchType, patch); err != nil {
+				return err
+			}
+
+			changed = true
+			return nil
+		}
+
+		return nil
+	}, func(serviceMonitor *monitoring.ServiceMonitor) bool {
+		return r.isChildResource(serviceMonitor)
+	}); err != nil {
+		return err
+	}
+
+	if changed {
+		return errors.Reconcile()
+	}
+
+	return nil
+}
+
 func (r *Resources) EnsurePodsLabels(cachedStatus inspector.Inspector) error {
 	changed := false
 	if err := cachedStatus.IteratePods(func(pod *core.Pod) error {

pkg/deployment/resources/secrets.go

@@ -186,7 +186,7 @@ func (r *Resources) refreshCache(cachedStatus inspector.Inspector, err error) er
 	}
 
 	if operatorErrors.IsReconcile(err) {
-		if err := cachedStatus.Refresh(r.context.GetKubeCli(), r.context.GetNamespace()); err != nil {
+		if err := cachedStatus.Refresh(r.context.GetKubeCli(), r.context.GetMonitoringV1Cli(), r.context.GetNamespace()); err != nil {
 			return maskAny(err)
 		}
 	} else {

pkg/operator/operator.go

@@ -27,6 +27,8 @@ import (
 	"math/rand"
 	"time"
 
+	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+
 	"github.com/arangodb/kube-arangodb/pkg/backup/operator/event"
 	"github.com/arangodb/kube-arangodb/pkg/util/constants"
 
@@ -121,6 +123,7 @@ type Dependencies struct {
 	LogService                 logging.Service
 	KubeCli                    kubernetes.Interface
 	KubeExtCli                 apiextensionsclient.Interface
+	KubeMonitoringCli          monitoringClient.MonitoringV1Interface
 	CRCli                      versioned.Interface
 	EventRecorder              record.EventRecorder
 	LivenessProbe              *probe.LivenessProbe

pkg/operator/operator_deployment.go

@@ -215,10 +215,11 @@ func (o *Operator) makeDeploymentConfigAndDeps(apiObject *api.ArangoDeployment)
 		Log: o.Dependencies.LogService.MustGetLogger("deployment").With().
 			Str("deployment", apiObject.GetName()).
 			Logger(),
-		KubeCli:       o.Dependencies.KubeCli,
-		KubeExtCli:    o.Dependencies.KubeExtCli,
-		DatabaseCRCli: o.Dependencies.CRCli,
-		EventRecorder: o.Dependencies.EventRecorder,
+		KubeCli:           o.Dependencies.KubeCli,
+		KubeMonitoringCli: o.Dependencies.KubeMonitoringCli,
+		KubeExtCli:        o.Dependencies.KubeExtCli,
+		DatabaseCRCli:     o.Dependencies.CRCli,
+		EventRecorder:     o.Dependencies.EventRecorder,
 	}
 	return cfg, deps
 }

pkg/util/k8sutil/client.go

@@ -26,6 +26,8 @@ import (
 	"fmt"
 	"os"
 
+	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+
 	"github.com/arangodb/kube-arangodb/pkg/util"
 	"k8s.io/client-go/tools/clientcmd"
 
@@ -93,3 +95,15 @@ func NewKubeExtClient() (apiextensionsclient.Interface, error) {
 	}
 	return c, nil
 }
+
+func NewKubeMonitoringV1Client() (monitoringClient.MonitoringV1Interface, error) {
+	cfg, err := NewKubeConfig()
+	if err != nil {
+		return nil, maskAny(err)
+	}
+	c, err := monitoringClient.NewForConfig(cfg)
+	if err != nil {
+		return nil, maskAny(err)
+	}
+	return c, nil
+}