feat: add metrics service and service monitor to cleanup controller (#5653)

* feat: add metrics service and service monitor to cleanup controller Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * more config Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * name and certs Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * fix: conditions check Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * leader election Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * workflows Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * rbac Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> Co-authored-by: Jim Bugwadia <jim@nirmata.com>
2025-03-31 03:45:17 +00:00 · 2022-12-12 19:39:29 +01:00 · 2022-12-12 19:39:29 +01:00 · f927b51b86
commit f927b51b86
parent 46003406b0
17 changed files with 289 additions and 64 deletions
--- a/.github/workflows/conformance.yaml
+++ b/.github/workflows/conformance.yaml
@ -35,5 +35,5 @@ jobs:
          kubectl -n kyverno describe pod | grep -i events -A10
          kubectl -n kyverno logs deploy/kyverno --all-containers -p || true
          kubectl -n kyverno logs deploy/kyverno --all-containers
-          kubectl -n kyverno logs deploy/cleanup-controller --all-containers -p || true
-          kubectl -n kyverno logs deploy/cleanup-controller --all-containers
+          kubectl -n kyverno logs deploy/kyverno-cleanup-controller --all-containers -p || true
+          kubectl -n kyverno logs deploy/kyverno-cleanup-controller --all-containers
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@ -89,6 +89,7 @@ jobs:
          kubectl get mutatingwebhookconfigurations,validatingwebhookconfigurations
          kubectl -n kyverno get pod
          kubectl -n kyverno describe pod | grep -i events -A10
-          kubectl -n kyverno logs deploy/kyverno -p || true
-          kubectl -n kyverno logs deploy/kyverno
-          kubectl -n kube-system logs kube-apiserver-kind-control-plane
+          kubectl -n kyverno logs deploy/kyverno --all-containers -p || true
+          kubectl -n kyverno logs deploy/kyverno --all-containers
+          kubectl -n kyverno logs deploy/kyverno-cleanup-controller --all-containers -p || true
+          kubectl -n kyverno logs deploy/kyverno-cleanup-controller --all-containers
--- a/charts/kyverno/README.md
+++ b/charts/kyverno/README.md
@ -229,7 +229,7 @@ The command removes all the Kubernetes components associated with the chart and
 | cleanupController.priorityClassName | string | `""` | Optional priority class |
 | cleanupController.hostNetwork | bool | `false` | Change `hostNetwork` to `true` when you want the pod to share its host's network namespace. Useful for situations like when you end up dealing with a custom CNI over Amazon EKS. Update the `dnsPolicy` accordingly as well to suit the host network mode. |
 | cleanupController.dnsPolicy | string | `"ClusterFirst"` | `dnsPolicy` determines the manner in which DNS resolution happens in the cluster. In case of `hostNetwork: true`, usually, the `dnsPolicy` is suitable to be `ClusterFirstWithHostNet`. For further reference: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy. |
-| cleanupController.args | list | `["--cleanupService=https://{{ template \"kyverno.cleanup-controller.deploymentName\" . }}.{{ template \"kyverno.namespace\" . }}.svc"]` | Arguments passed to the container on the command line |
+| cleanupController.extraArgs | list | `[]` | Extra arguments passed to the container on the command line |
 | cleanupController.resources.limits | object | `{"memory":"128Mi"}` | Pod resource limits |
 | cleanupController.resources.requests | object | `{"cpu":"100m","memory":"64Mi"}` | Pod resource requests |
 | cleanupController.startupProbe | object | See [values.yaml](values.yaml) | Startup probe. The block is directly forwarded into the deployment, so you can use whatever startupProbes configuration you want. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ |
@ -250,6 +250,28 @@ The command removes all the Kubernetes components associated with the chart and
 | cleanupController.service.type | string | `"ClusterIP"` | Service type. |
 | cleanupController.service.nodePort | string | `nil` | Service node port. Only used if `service.type` is `NodePort`. |
 | cleanupController.service.annotations | object | `{}` | Service annotations. |
+| cleanupController.metricsService.create | bool | `true` | Create service. |
+| cleanupController.metricsService.port | int | `8000` | Service port. Metrics server will be exposed at this port. |
+| cleanupController.metricsService.type | string | `"ClusterIP"` | Service type. |
+| cleanupController.metricsService.nodePort | string | `nil` | Service node port. Only used if `metricsService.type` is `NodePort`. |
+| cleanupController.metricsService.annotations | object | `{}` | Service annotations. |
+| cleanupController.serviceMonitor.enabled | bool | `false` | Create a `ServiceMonitor` to collect Prometheus metrics. |
+| cleanupController.serviceMonitor.additionalLabels | string | `nil` | Additional labels |
+| cleanupController.serviceMonitor.namespace | string | `nil` | Override namespace (default is the same as kyverno) |
+| cleanupController.serviceMonitor.interval | string | `"30s"` | Interval to scrape metrics |
+| cleanupController.serviceMonitor.scrapeTimeout | string | `"25s"` | Timeout if metrics can't be retrieved in given time interval |
+| cleanupController.serviceMonitor.secure | bool | `false` | Is TLS required for endpoint |
+| cleanupController.serviceMonitor.tlsConfig | object | `{}` | TLS Configuration for endpoint |
+| cleanupController.tracing.enabled | bool | `false` | Enable tracing |
+| cleanupController.tracing.address | string | `nil` | Traces receiver address |
+| cleanupController.tracing.port | string | `nil` | Traces receiver port |
+| cleanupController.tracing.creds | string | `""` | Traces receiver credentials |
+| cleanupController.logging.format | string | `"text"` | Logging format |
+| cleanupController.metering.disabled | bool | `false` | Disable metrics export |
+| cleanupController.metering.config | string | `"prometheus"` | Otel configuration, can be `prometheus` or `grpc` |
+| cleanupController.metering.port | int | `8000` | Prometheus endpoint port |
+| cleanupController.metering.collector | string | `""` | Otel collector endpoint |
+| cleanupController.metering.creds | string | `""` | Otel collector credentials |

 ## TLS Configuration

--- a/charts/kyverno/templates/cleanup-controller/_helpers.tpl
+++ b/charts/kyverno/templates/cleanup-controller/_helpers.tpl
@ -1,14 +1,12 @@
 {{/* vim: set filetype=mustache: */}}

-{{- define "kyverno.cleanup-controller.deploymentName" -}}
-cleanup-controller
+{{- define "kyverno.cleanup-controller.name" -}}
+{{ template "kyverno.name" . }}-cleanup-controller
 {{- end -}}

 {{- define "kyverno.cleanup-controller.labels" -}}
-app.kubernetes.io/component: cleanup-controller
-app.kubernetes.io/instance: {{ .Release.Name }}
+{{- include "kyverno.cleanup-controller.matchLabels" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
-app.kubernetes.io/name: {{ template "kyverno.name" . }}
 app.kubernetes.io/part-of: {{ template "kyverno.name" . }}
 app.kubernetes.io/version: "{{ .Chart.Version | replace "+" "_" }}"
 helm.sh/chart: {{ template "kyverno.chart" . }}
@ -16,7 +14,7 @@ helm.sh/chart: {{ template "kyverno.chart" . }}

 {{- define "kyverno.cleanup-controller.matchLabels" -}}
 app.kubernetes.io/component: cleanup-controller
-app.kubernetes.io/name: {{ template "kyverno.name" . }}
+app.kubernetes.io/name: {{ template "kyverno.cleanup-controller.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end -}}

@ -31,7 +29,7 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 {{/* Create the name of the service account to use */}}
 {{- define "kyverno.cleanup-controller.serviceAccountName" -}}
 {{- if .Values.cleanupController.rbac.create -}}
-    {{ default (include "kyverno.cleanup-controller.deploymentName" .) .Values.cleanupController.rbac.serviceAccount.name }}
+    {{ default (include "kyverno.cleanup-controller.name" .) .Values.cleanupController.rbac.serviceAccount.name }}
 {{- else -}}
    {{ required "A service account name is required when `rbac.create` is set to `false`" .Values.cleanupController.rbac.serviceAccount.name }}
 {{- end -}}
--- a/charts/kyverno/templates/cleanup-controller/clusterrole.yaml
+++ b/charts/kyverno/templates/cleanup-controller/clusterrole.yaml
@ -3,7 +3,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: {{ template "kyverno.cleanup-controller.deploymentName" . }}
+  name: {{ template "kyverno.cleanup-controller.name" . }}
  labels:
    {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
 rules:
--- a/charts/kyverno/templates/cleanup-controller/clusterrolebinding.yaml
+++ b/charts/kyverno/templates/cleanup-controller/clusterrolebinding.yaml
@ -3,13 +3,13 @@
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: {{ template "kyverno.cleanup-controller.deploymentName" . }}
+  name: {{ template "kyverno.cleanup-controller.name" . }}
  labels:
    {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
-  name: {{ template "kyverno.cleanup-controller.deploymentName" . }}
+  name: {{ template "kyverno.cleanup-controller.name" . }}
 subjects:
 - kind: ServiceAccount
  name: {{ template "kyverno.cleanup-controller.serviceAccountName" . }}
--- a/charts/kyverno/templates/cleanup-controller/deployment.yaml
+++ b/charts/kyverno/templates/cleanup-controller/deployment.yaml
@ -2,7 +2,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ template "kyverno.cleanup-controller.deploymentName" . }}
+  name: {{ template "kyverno.cleanup-controller.name" . }}
  labels:
    {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
  namespace: {{ template "kyverno.namespace" . }}
@ -77,17 +77,43 @@ spec:
          - containerPort: 8000
            name: metrics
            protocol: TCP
-          {{- with .Values.cleanupController.args }}
          args:
-            {{- tpl (toYaml .) $ | nindent 12 }}
-          {{- end }}
+            - --loggingFormat={{ .Values.cleanupController.logging.format }}
+            {{- if .Values.cleanupController.tracing.enabled }}
+            - --enableTracing
+            - --tracingAddress={{ .Values.cleanupController.tracing.address }}
+            - --tracingPort={{ .Values.cleanupController.tracing.port }}
+            {{- with .Values.cleanupController.tracing.creds }}
+            - --tracingCreds={{ . }}
+            {{- end }}
+            {{- end }}
+            - --disableMetrics={{ .Values.cleanupController.metering.disabled }}
+            {{- if not .Values.cleanupController.metering.disabled }}
+            - --otelConfig={{ .Values.cleanupController.metering.config }}
+            - --metricsPort={{ .Values.cleanupController.metering.port }}
+            {{- with .Values.cleanupController.metering.collector }}
+            - --otelCollector={{ . }}
+            {{- end }}
+            {{- with .Values.cleanupController.metering.creds }}
+            - --transportCreds={{ . }}
+            {{- end }}
+            {{- end }}
+            {{- range .Values.cleanupController.extraArgs }}
+            - {{ . }}
+            {{- end }}
          env:
          - name: METRICS_CONFIG
            value: {{ template "kyverno.metricsConfigMapName" . }}
+          - name: KYVERNO_POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
          - name: KYVERNO_NAMESPACE
            valueFrom:
              fieldRef:
                fieldPath: metadata.namespace
+          - name: KYVERNO_SVC
+            value: {{ template "kyverno.cleanup-controller.name" . }}
          {{- with .Values.cleanupController.resources }}
          resources: {{ tpl (toYaml .) $ | nindent 12 }}
          {{- end }}
--- a/charts/kyverno/templates/cleanup-controller/poddisruptionbudget.yaml
+++ b/charts/kyverno/templates/cleanup-controller/poddisruptionbudget.yaml
@ -7,7 +7,7 @@ apiVersion: policy/v1beta1
 {{- end }}
 kind: PodDisruptionBudget
 metadata:
-  name: {{ template "kyverno.cleanup-controller.deploymentName" . }}
+  name: {{ template "kyverno.cleanup-controller.name" . }}
  labels:
    {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
  namespace: {{ template "kyverno.namespace" . }}
--- a/charts/kyverno/templates/cleanup-controller/role.yaml
+++ b/charts/kyverno/templates/cleanup-controller/role.yaml
@ -3,7 +3,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: {{ template "kyverno.cleanup-controller.deploymentName" . }}
+  name: {{ template "kyverno.cleanup-controller.name" . }}
  labels:
    {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
  namespace: {{ template "kyverno.namespace" . }}
@ -17,5 +17,15 @@ rules:
    - get
    - list
    - watch
+- apiGroups:
+    - coordination.k8s.io
+  resources:
+    - leases
+  verbs:
+    - create
+    - delete
+    - get
+    - patch
+    - update
 {{- end -}}
 {{- end -}}
--- a/charts/kyverno/templates/cleanup-controller/rolebinding.yaml
+++ b/charts/kyverno/templates/cleanup-controller/rolebinding.yaml
@ -3,17 +3,17 @@
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: {{ template "kyverno.cleanup-controller.deploymentName" . }}
+  name: {{ template "kyverno.cleanup-controller.name" . }}
  labels:
    {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
  namespace: {{ template "kyverno.namespace" . }}
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
-  name: {{ template "kyverno.cleanup-controller.deploymentName" . }}
+  name: {{ template "kyverno.cleanup-controller.name" . }}
 subjects:
 - kind: ServiceAccount
-  name: {{ template "kyverno.cleanup-controller.deploymentName" . }}
+  name: {{ template "kyverno.cleanup-controller.serviceAccountName" . }}
  namespace: {{ template "kyverno.namespace" . }}
 {{- end -}}
 {{- end -}}
--- a/charts/kyverno/templates/cleanup-controller/secret.yaml
+++ b/charts/kyverno/templates/cleanup-controller/secret.yaml
@ -1,11 +1,11 @@
 {{- if .Values.cleanupController.enabled -}}
 {{- $ca := genCA (printf "*.%s.svc" (include "kyverno.namespace" .)) 1024 -}}
-{{- $svcName := (printf "%s.%s.svc" (include "kyverno.cleanup-controller.deploymentName" .) (include "kyverno.namespace" .)) -}}
+{{- $svcName := (printf "%s.%s.svc" (include "kyverno.cleanup-controller.name" .) (include "kyverno.namespace" .)) -}}
 {{- $cert := genSignedCert $svcName nil (list $svcName) 1024 $ca -}}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ template "kyverno.cleanup-controller.deploymentName" . }}-ca
+  name: {{ template "kyverno.cleanup-controller.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-ca
  labels:
    {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
  namespace: {{ template "kyverno.namespace" . }}
@ -17,7 +17,7 @@ data:
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ template "kyverno.cleanup-controller.deploymentName" . }}-tls
+  name: {{ template "kyverno.cleanup-controller.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-pair
  labels:
    {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
  namespace: {{ template "kyverno.namespace" . }}
@ -29,7 +29,7 @@ data:
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
-  name: {{ template "kyverno.cleanup-controller.deploymentName" . }}
+  name: {{ template "kyverno.cleanup-controller.name" . }}
  labels:
    {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
 webhooks:
@ -38,13 +38,13 @@ webhooks:
  clientConfig:
    caBundle:  {{ $ca.Cert | b64enc }}
    service:
-      name: {{ template "kyverno.cleanup-controller.deploymentName" . }}
+      name: {{ template "kyverno.cleanup-controller.name" . }}
      namespace: {{ template "kyverno.namespace" . }}
      path: /validate
      port: 443
  failurePolicy: Fail
  matchPolicy: Equivalent
-  name: {{ printf "%s.%s.svc" (include "kyverno.cleanup-controller.deploymentName" .) (include "kyverno.namespace" .) }}
+  name: {{ printf "%s.%s.svc" (include "kyverno.cleanup-controller.name" .) (include "kyverno.namespace" .) }}
  rules:
  - apiGroups:
    - kyverno.io
--- a/charts/kyverno/templates/cleanup-controller/service.yaml
+++ b/charts/kyverno/templates/cleanup-controller/service.yaml
@ -2,10 +2,14 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ template "kyverno.cleanup-controller.deploymentName" . }}
+  name: {{ template "kyverno.cleanup-controller.name" . }}
+  namespace: {{ template "kyverno.namespace" . }}
  labels:
    {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
-  namespace: {{ template "kyverno.namespace" . }}
+  {{- with .Values.cleanupController.service.annotations }}
+  annotations:
+    {{- tpl (toYaml .) $ | nindent 4 }}
+  {{- end }}
 spec:
  ports:
  - port: {{ .Values.cleanupController.service.port }}
@ -18,4 +22,30 @@ spec:
  selector:
    {{- include "kyverno.cleanup-controller.matchLabels" . | nindent 4 }}
  type: {{ .Values.cleanupController.service.type }}
+{{- if .Values.cleanupController.metricsService.create }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "kyverno.cleanup-controller.name" . }}-metrics
+  namespace: {{ template "kyverno.namespace" . }}
+  labels:
+    {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
+  {{- with .Values.cleanupController.metricsService.annotations }}
+  annotations:
+    {{- tpl (toYaml .) $ | nindent 4 }}
+  {{- end }}
+spec:
+  ports:
+  - port: {{ .Values.cleanupController.metricsService.port }}
+    targetPort: 8000
+    protocol: TCP
+    name: metrics-port
+    {{- if and (eq .Values.cleanupController.metricsService.type "NodePort") (not (empty .Values.cleanupController.metricsService.nodePort)) }}
+    nodePort: {{ .Values.cleanupController.metricsService.nodePort }}
+    {{- end }}
+  selector:
+    {{- include "kyverno.cleanup-controller.matchLabels" . | nindent 4 }}
+  type: {{ .Values.cleanupController.metricsService.type }}
+{{- end -}}
 {{- end -}}
--- a/charts/kyverno/templates/cleanup-controller/servicemonitor.yaml
+++ b/charts/kyverno/templates/cleanup-controller/servicemonitor.yaml
@ -0,0 +1,34 @@
+{{- if .Values.cleanupController.enabled -}}
+{{- if .Values.cleanupController.serviceMonitor.enabled -}}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "kyverno.cleanup-controller.name" . }}
+  {{- if .Values.cleanupController.serviceMonitor.namespace }}
+  namespace: {{ .Values.cleanupController.serviceMonitor.namespace }}
+  {{- else }}
+  namespace: {{ template "kyverno.namespace" . }}
+  {{- end }}
+  labels:
+    {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
+    {{- with .Values.cleanupController.serviceMonitor.additionalLabels }}
+    {{- toYaml .Values.cleanupController.serviceMonitor.additionalLabels | nindent 4 }}
+    {{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "kyverno.cleanup-controller.matchLabels" . | nindent 6 }}
+  namespaceSelector:
+    matchNames:
+    - {{ template "kyverno.namespace" . }}
+  endpoints:
+  - port: metrics-port
+    interval: {{ .Values.cleanupController.serviceMonitor.interval }}
+    scrapeTimeout: {{ .Values.cleanupController.serviceMonitor.scrapeTimeout }}
+    {{- if .Values.cleanupController.serviceMonitor.secure }}
+    scheme: https
+    tlsConfig:
+      {{- toYaml .Values.cleanupController.serviceMonitor.tlsConfig | nindent 8 }}
+    {{- end }}
+{{- end -}}
+{{- end -}}
--- a/charts/kyverno/values.yaml
+++ b/charts/kyverno/values.yaml
@ -548,9 +548,8 @@ cleanupController:
  # For further reference: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy.
  dnsPolicy: ClusterFirst

-  # -- Arguments passed to the container on the command line
-  args:
-    - --cleanupService=https://{{ template "kyverno.cleanup-controller.deploymentName" . }}.{{ template "kyverno.namespace" . }}.svc
+  # -- Extra arguments passed to the container on the command line
+  extraArgs: []

  resources:
    # -- Pod resource limits
@ -671,3 +670,60 @@ cleanupController:
    nodePort:
    # -- Service annotations.
    annotations: {}
+
+  metricsService:
+    # -- Create service.
+    create: true
+    # -- Service port.
+    # Metrics server will be exposed at this port.
+    port: 8000
+    # -- Service type.
+    type: ClusterIP
+    # -- Service node port.
+    # Only used if `metricsService.type` is `NodePort`.
+    nodePort:
+    # -- Service annotations.
+    annotations: {}
+
+  serviceMonitor:
+    # -- Create a `ServiceMonitor` to collect Prometheus metrics.
+    enabled: false
+    # -- Additional labels
+    additionalLabels:
+      # key: value
+    # -- Override namespace (default is the same as kyverno)
+    namespace:
+    # --  Interval to scrape metrics
+    interval: 30s
+    # -- Timeout if metrics can't be retrieved in given time interval
+    scrapeTimeout: 25s
+    # -- Is TLS required for endpoint
+    secure: false
+    # -- TLS Configuration for endpoint
+    tlsConfig: {}
+
+  tracing:
+    # -- Enable tracing
+    enabled: false
+    # -- Traces receiver address
+    address:
+    # -- Traces receiver port
+    port:
+    # -- Traces receiver credentials
+    creds: ''
+
+  logging:
+    # -- Logging format
+    format: text
+
+  metering:
+    # -- Disable metrics export
+    disabled: false
+    # -- Otel configuration, can be `prometheus` or `grpc`
+    config: prometheus
+    # -- Prometheus endpoint port
+    port: 8000
+    # -- Otel collector endpoint
+    collector: ''
+    # -- Otel collector credentials
+    creds: ''
--- a/cmd/cleanup-controller/handlers/cleanup/condition.go
+++ b/cmd/cleanup-controller/handlers/cleanup/condition.go
@ -25,7 +25,7 @@ func checkAnyAllConditions(logger logr.Logger, ctx enginecontext.Interface, cond
 			return true, nil
 		}
 	}
-	return true, nil
+	return len(condition.AnyConditions) == 0, nil
 }

 func checkCondition(logger logr.Logger, ctx enginecontext.Interface, condition kyvernov2beta1.Condition) (bool, error) {
--- a/cmd/cleanup-controller/main.go
+++ b/cmd/cleanup-controller/main.go
@ -1,6 +1,8 @@
 package main

 import (
+	"context"
+	"errors"
 	"flag"
 	"os"
 	"sync"
@ -15,7 +17,9 @@ import (
 	kyvernoclient "github.com/kyverno/kyverno/pkg/clients/kyverno"
 	"github.com/kyverno/kyverno/pkg/config"
 	"github.com/kyverno/kyverno/pkg/controllers/cleanup"
+	"github.com/kyverno/kyverno/pkg/leaderelection"
 	"github.com/kyverno/kyverno/pkg/metrics"
+	"github.com/kyverno/kyverno/pkg/tls"
 	"github.com/kyverno/kyverno/pkg/webhooks"
 	corev1 "k8s.io/api/core/v1"
 	kubeinformers "k8s.io/client-go/informers"
@ -26,14 +30,10 @@ const (
 )

 // TODO:
+// - helm review labels / selectors
 // - implement probes
 // - better certs management
 // - supports certs in cronjob
-// - leader election support
-// - helm service monitor
-// - helm name and fullname
-// - helm review labels / selectors
-// - helm metrics service

 type probes struct{}

@ -47,12 +47,12 @@ func (probes) IsLive() bool {

 func main() {
 	var (
-		cleanupService string
-		dumpPayload    bool
+		leaderElectionRetryPeriod time.Duration
+		dumpPayload               bool
 	)
 	flagset := flag.NewFlagSet("cleanup-controller", flag.ExitOnError)
-	flagset.StringVar(&cleanupService, "cleanupService", "https://cleanup-controller.kyverno.svc", "The url to join the cleanup service.")
 	flagset.BoolVar(&dumpPayload, "dumpPayload", false, "Set this flag to activate/deactivate debug mode.")
+	flagset.DurationVar(&leaderElectionRetryPeriod, "leaderElectionRetryPeriod", leaderelection.DefaultRetryPeriod, "Configure leader election retry period.")
 	// config
 	appConfig := internal.NewConfiguration(
 		internal.WithProfiling(),
@ -73,26 +73,58 @@ func main() {
 	defer sdown()
 	// create instrumented clients
 	kubeClient := internal.CreateKubernetesClient(logger, kubeclient.WithMetrics(metricsConfig, metrics.KubeClient), kubeclient.WithTracing())
-	dynamicClient := internal.CreateDynamicClient(logger, dynamicclient.WithMetrics(metricsConfig, metrics.KyvernoClient), dynamicclient.WithTracing())
+	leaderElectionClient := internal.CreateKubernetesClient(logger, kubeclient.WithMetrics(metricsConfig, metrics.KubeClient), kubeclient.WithTracing())
 	kyvernoClient := internal.CreateKyvernoClient(logger, kyvernoclient.WithMetrics(metricsConfig, metrics.KubeClient), kyvernoclient.WithTracing())
+	// setup leader election
+	le, err := leaderelection.New(
+		logger.WithName("leader-election"),
+		"kyverno-cleanup-controller",
+		config.KyvernoNamespace(),
+		leaderElectionClient,
+		config.KyvernoPodName(),
+		leaderElectionRetryPeriod,
+		func(ctx context.Context) {
+			logger := logger.WithName("leader")
+			// informer factories
+			kubeInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, resyncPeriod)
+			kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(kyvernoClient, resyncPeriod)
+			// controllers
+			controller := internal.NewController(
+				cleanup.ControllerName,
+				cleanup.NewController(
+					kubeClient,
+					kyvernoInformer.Kyverno().V2alpha1().ClusterCleanupPolicies(),
+					kyvernoInformer.Kyverno().V2alpha1().CleanupPolicies(),
+					kubeInformer.Batch().V1().CronJobs(),
+					"https://"+config.KyvernoServiceName()+"."+config.KyvernoNamespace()+".svc",
+				),
+				cleanup.Workers,
+			)
+			// start informers and wait for cache sync
+			if !internal.StartInformersAndWaitForCacheSync(ctx, kyvernoInformer, kubeInformer) {
+				logger.Error(errors.New("failed to wait for cache sync"), "failed to wait for cache sync")
+				os.Exit(1)
+			}
+			// start leader controllers
+			var wg sync.WaitGroup
+			controller.Run(ctx, logger.WithName("cleanup-controller"), &wg)
+			// wait all controllers shut down
+			wg.Wait()
+		},
+		nil,
+	)
+	if err != nil {
+		logger.Error(err, "failed to initialize leader election")
+		os.Exit(1)
+	}
+	dynamicClient := internal.CreateDynamicClient(logger, dynamicclient.WithMetrics(metricsConfig, metrics.KyvernoClient), dynamicclient.WithTracing())
 	dClient := internal.CreateDClient(logger, ctx, dynamicClient, kubeClient, 15*time.Minute)
 	// informer factories
 	kubeInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, resyncPeriod)
 	kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace()))
 	kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(kyvernoClient, resyncPeriod)
-	// controllers
-	controller := internal.NewController(
-		cleanup.ControllerName,
-		cleanup.NewController(
-			kubeClient,
-			kyvernoInformer.Kyverno().V2alpha1().ClusterCleanupPolicies(),
-			kyvernoInformer.Kyverno().V2alpha1().CleanupPolicies(),
-			kubeInformer.Batch().V1().CronJobs(),
-			cleanupService,
-		),
-		cleanup.Workers,
-	)
-	secretLister := kubeKyvernoInformer.Core().V1().Secrets().Lister()
+	// listers
+	secretLister := kubeKyvernoInformer.Core().V1().Secrets().Lister().Secrets(config.KyvernoNamespace())
 	cpolLister := kyvernoInformer.Kyverno().V2alpha1().ClusterCleanupPolicies().Lister()
 	polLister := kyvernoInformer.Kyverno().V2alpha1().CleanupPolicies().Lister()
 	nsLister := kubeInformer.Core().V1().Namespaces().Lister()
@ -100,15 +132,13 @@ func main() {
 	if !internal.StartInformersAndWaitForCacheSync(ctx, kubeKyvernoInformer, kubeInformer, kyvernoInformer) {
 		os.Exit(1)
 	}
-	var wg sync.WaitGroup
-	controller.Run(ctx, logger.WithName("cleanup-controller"), &wg)
 	// create handlers
 	admissionHandlers := admissionhandlers.New(dClient)
 	cleanupHandlers := cleanuphandlers.New(dClient, cpolLister, polLister, nsLister)
 	// create server
 	server := NewServer(
 		func() ([]byte, []byte, error) {
-			secret, err := secretLister.Secrets(config.KyvernoNamespace()).Get("cleanup-controller-tls")
+			secret, err := secretLister.Get(tls.GenerateTLSPairSecretName())
 			if err != nil {
 				return nil, nil, err
 			}
@ -124,6 +154,13 @@ func main() {
 	)
 	// start server
 	server.Run(ctx.Done())
-	// wait for termination signal
-	wg.Wait()
+	// wait for termination signal and run leader election loop
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			le.Run(ctx)
+		}
+	}
 }
--- a/scripts/config/dev/kyverno.yaml
+++ b/scripts/config/dev/kyverno.yaml
@ -22,3 +22,14 @@ cleanupController:
            - ''
          resources:
            - pods
+
+  serviceMonitor:
+    enabled: true
+
+  tracing:
+    enabled: true
+    address: tempo.monitoring
+    port: 4317
+
+  logging:
+    format: json