From 084336c5f56da502c113aa38d5e2b09e064093df Mon Sep 17 00:00:00 2001 From: shuting Date: Wed, 29 May 2024 18:23:54 +0800 Subject: [PATCH] feat: add a cleanup cronjob to delete urs (#10249) Signed-off-by: ShutingZhao --- charts/kyverno/README.md | 22 +++++ .../cleanup/cleanup-update-requests.yaml | 91 +++++++++++++++++++ .../templates/cleanup/clusterrole.yaml | 1 + charts/kyverno/values.yaml | 80 ++++++++++++++++ config/install-latest-testing.yaml | 51 +++++++++++ 5 files changed, 245 insertions(+) create mode 100644 charts/kyverno/templates/cleanup/cleanup-update-requests.yaml diff --git a/charts/kyverno/README.md b/charts/kyverno/README.md index 4347065cd9..84ff298444 100644 --- a/charts/kyverno/README.md +++ b/charts/kyverno/README.md @@ -771,6 +771,28 @@ The chart values are organised per component. | cleanupJobs.clusterAdmissionReports.podAntiAffinity | object | `{}` | Pod anti affinity constraints. | | cleanupJobs.clusterAdmissionReports.podAffinity | object | `{}` | Pod affinity constraints. | | cleanupJobs.clusterAdmissionReports.nodeAffinity | object | `{}` | Node affinity constraints. | +| cleanupJobs.updateRequests.enabled | bool | `true` | Enable cleanup cronjob | +| cleanupJobs.updateRequests.backoffLimit | int | `3` | Maximum number of retries before considering a Job as failed. Defaults to 3. | +| cleanupJobs.updateRequests.ttlSecondsAfterFinished | string | `""` | Time until the pod from the cronjob is deleted | +| cleanupJobs.updateRequests.image.registry | string | `nil` | Image registry | +| cleanupJobs.updateRequests.image.repository | string | `"bitnami/kubectl"` | Image repository | +| cleanupJobs.updateRequests.image.tag | string | `"1.28.5"` | Image tag Defaults to `latest` if omitted | +| cleanupJobs.updateRequests.image.pullPolicy | string | `nil` | Image pull policy Defaults to image.pullPolicy if omitted | +| cleanupJobs.updateRequests.imagePullSecrets | list | `[]` | Image pull secrets | +| cleanupJobs.updateRequests.schedule | string | `"*/10 * * * *"` | Cronjob schedule | +| cleanupJobs.updateRequests.threshold | int | `10000` | Reports threshold, if number of updateRequests are above this value the cronjob will start deleting them | +| cleanupJobs.updateRequests.history | object | `{"failure":1,"success":1}` | Cronjob history | +| cleanupJobs.updateRequests.podSecurityContext | object | `{}` | Security context for the pod | +| cleanupJobs.updateRequests.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}` | Security context for the containers | +| cleanupJobs.updateRequests.priorityClassName | string | `""` | Pod PriorityClassName | +| cleanupJobs.updateRequests.resources | object | `{}` | Job resources | +| cleanupJobs.updateRequests.tolerations | list | `[]` | List of node taints to tolerate | +| cleanupJobs.updateRequests.nodeSelector | object | `{}` | Node labels for pod assignment | +| cleanupJobs.updateRequests.podAnnotations | object | `{}` | Pod Annotations | +| cleanupJobs.updateRequests.podLabels | object | `{}` | Pod labels | +| cleanupJobs.updateRequests.podAntiAffinity | object | `{}` | Pod anti affinity constraints. | +| cleanupJobs.updateRequests.podAffinity | object | `{}` | Pod affinity constraints. | +| cleanupJobs.updateRequests.nodeAffinity | object | `{}` | Node affinity constraints. | ### Other diff --git a/charts/kyverno/templates/cleanup/cleanup-update-requests.yaml b/charts/kyverno/templates/cleanup/cleanup-update-requests.yaml new file mode 100644 index 0000000000..ac564f9377 --- /dev/null +++ b/charts/kyverno/templates/cleanup/cleanup-update-requests.yaml @@ -0,0 +1,91 @@ +{{- if .Values.cleanupJobs.updateRequests.enabled -}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ template "kyverno.name" . }}-cleanup-update-requests + namespace: {{ template "kyverno.namespace" . }} + labels: + {{- include "kyverno.cleanup.labels" . | nindent 4 }} +spec: + schedule: {{ .Values.cleanupJobs.updateRequests.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: {{ .Values.cleanupJobs.updateRequests.history.success }} + failedJobsHistoryLimit: {{ .Values.cleanupJobs.updateRequests.history.failure }} + jobTemplate: + spec: + backoffLimit: {{ .Values.cleanupJobs.updateRequests.backoffLimit }} + {{- if .Values.cleanupJobs.updateRequests.ttlSecondsAfterFinished }} + ttlSecondsAfterFinished: {{ .Values.cleanupJobs.updateRequests.ttlSecondsAfterFinished }} + {{- end }} + template: + metadata: + {{- with .Values.cleanupJobs.updateRequests.podAnnotations }} + annotations: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.cleanupJobs.updateRequests.podLabels }} + labels: + {{- toYaml . | nindent 12 }} + {{- end }} + spec: + serviceAccountName: {{ template "kyverno.name" . }}-cleanup-jobs + {{- with .Values.cleanupJobs.updateRequests.podSecurityContext }} + securityContext: + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- with .Values.cleanupJobs.updateRequests.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + containers: + - name: cleanup + image: {{ (include "kyverno.image" (dict "globalRegistry" .Values.global.image.registry "image" .Values.cleanupJobs.updateRequests.image)) | quote }} + imagePullPolicy: {{ .Values.cleanupJobs.updateRequests.image.pullPolicy }} + command: + - /bin/bash + - -c + - | + set -euo pipefail + COUNT=$(kubectl get updaterequests.kyverno.io -A | wc -l) + if [ "$COUNT" -gt {{ .Values.cleanupJobs.updateRequests.threshold }} ]; then + echo "too many updaterequests found ($COUNT), cleaning up..." + kubectl delete updaterequests.kyverno.io --all -n kyverno + else + echo "($COUNT) reports found, no clean up needed" + fi + {{- with .Values.cleanupJobs.updateRequests.securityContext }} + securityContext: + {{- toYaml . | nindent 14 }} + {{- end }} + {{- with .Values.cleanupJobs.updateRequests.resources }} + resources: + {{- toYaml . | nindent 14 }} + {{- end }} + {{- with .Values.cleanupJobs.updateRequests.imagePullSecrets }} + imagePullSecrets: + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + restartPolicy: OnFailure + {{- with .Values.cleanupJobs.updateRequests.tolerations }} + tolerations: + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- with .Values.cleanupJobs.updateRequests.nodeSelector | default .Values.global.nodeSelector }} + nodeSelector: + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- if or .Values.cleanupJobs.updateRequests.podAntiAffinity .Values.cleanupJobs.updateRequests.podAffinity .Values.cleanupJobs.admissionReports.nodeAffinity }} + affinity: + {{- with .Values.cleanupJobs.updateRequests.podAntiAffinity }} + podAntiAffinity: + {{- tpl (toYaml .) $ | nindent 14 }} + {{- end }} + {{- with .Values.cleanupJobs.updateRequests.podAffinity }} + podAffinity: + {{- tpl (toYaml .) $ | nindent 14 }} + {{- end }} + {{- with .Values.cleanupJobs.updateRequests.nodeAffinity }} + nodeAffinity: + {{- tpl (toYaml .) $ | nindent 14 }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/kyverno/templates/cleanup/clusterrole.yaml b/charts/kyverno/templates/cleanup/clusterrole.yaml index 1c570fcb24..3f7888be67 100644 --- a/charts/kyverno/templates/cleanup/clusterrole.yaml +++ b/charts/kyverno/templates/cleanup/clusterrole.yaml @@ -10,6 +10,7 @@ rules: resources: - admissionreports - clusteradmissionreports + - updaterequests verbs: - list - deletecollection diff --git a/charts/kyverno/values.yaml b/charts/kyverno/values.yaml index 818b2854e5..e25b921eff 100644 --- a/charts/kyverno/values.yaml +++ b/charts/kyverno/values.yaml @@ -857,6 +857,86 @@ cleanupJobs: # -- Node affinity constraints. nodeAffinity: {} + updateRequests: + + # -- Enable cleanup cronjob + enabled: true + + # -- Maximum number of retries before considering a Job as failed. Defaults to 3. + backoffLimit: 3 + + # -- Time until the pod from the cronjob is deleted + ttlSecondsAfterFinished: "" + + image: + # -- (string) Image registry + registry: ~ + # -- Image repository + repository: bitnami/kubectl + # -- Image tag + # Defaults to `latest` if omitted + tag: '1.28.5' + # -- (string) Image pull policy + # Defaults to image.pullPolicy if omitted + pullPolicy: ~ + + # -- Image pull secrets + imagePullSecrets: [] + # - name: secretName + + # -- Cronjob schedule + schedule: '*/10 * * * *' + + # -- Reports threshold, if number of updateRequests are above this value the cronjob will start deleting them + threshold: 10000 + + # -- Cronjob history + history: + success: 1 + failure: 1 + + # -- Security context for the pod + podSecurityContext: {} + + # -- Security context for the containers + securityContext: + runAsNonRoot: true + privileged: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + + # -- Pod PriorityClassName + priorityClassName: "" + + # -- Job resources + resources: {} + + # -- List of node taints to tolerate + tolerations: [] + + # -- Node labels for pod assignment + nodeSelector: {} + + # -- Pod Annotations + podAnnotations: {} + + # -- Pod labels + podLabels: {} + + # -- Pod anti affinity constraints. + podAntiAffinity: {} + + # -- Pod affinity constraints. + podAffinity: {} + + # -- Node affinity constraints. + nodeAffinity: {} + # Admission controller configuration admissionController: diff --git a/config/install-latest-testing.yaml b/config/install-latest-testing.yaml index 9e2fc9ba70..f16bdce271 100644 --- a/config/install-latest-testing.yaml +++ b/config/install-latest-testing.yaml @@ -48797,6 +48797,7 @@ rules: resources: - admissionreports - clusteradmissionreports + - updaterequests verbs: - list - deletecollection @@ -50280,3 +50281,53 @@ spec: seccompProfile: type: RuntimeDefault restartPolicy: OnFailure +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: kyverno-cleanup-update-requests + namespace: kyverno + labels: + app.kubernetes.io/component: cleanup + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno + app.kubernetes.io/version: latest +spec: + schedule: "*/10 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 3 + template: + metadata: + spec: + serviceAccountName: kyverno-cleanup-jobs + containers: + - name: cleanup + image: "bitnami/kubectl:1.28.5" + imagePullPolicy: + command: + - /bin/bash + - -c + - | + set -euo pipefail + COUNT=$(kubectl get updaterequests.kyverno.io -A | wc -l) + if [ "$COUNT" -gt 10000 ]; then + echo "too many updaterequests found ($COUNT), cleaning up..." + kubectl delete updaterequests.kyverno.io --all -n kyverno + else + echo "($COUNT) reports found, no clean up needed" + fi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + restartPolicy: OnFailure