feat: add reports cleanup jobs to prevent outage (#6960)

* feat: add reports cleanup jobs to prevent outage

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* security cotnext

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Co-authored-by: shuting <shuting@nirmata.com>

charts/kyverno/README.md

@@ -557,6 +557,31 @@ The chart values are organised per component.
 |-----|------|---------|-------------|
 | apiVersionOverride.podDisruptionBudget | string | `nil` | Override api version used to create `PodDisruptionBudget`` resources. When not specified the chart will check if `policy/v1/PodDisruptionBudget` is available to determine the api version automatically. |
 
+### Cleanup jobs
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| cleanupJobs.admissionReports.enabled | bool | `true` | Enable cleanup cronjob |
+| cleanupJobs.admissionReports.image.registry | string | `nil` | Image registry |
+| cleanupJobs.admissionReports.image.repository | string | `"bitnami/kubectl"` | Image repository |
+| cleanupJobs.admissionReports.image.tag | string | `"1.26.4"` | Image tag Defaults to `latest` if omitted |
+| cleanupJobs.admissionReports.image.pullPolicy | string | `nil` | Image pull policy Defaults to image.pullPolicy if omitted |
+| cleanupJobs.admissionReports.schedule | string | `"*/10 * * * *"` | Cronjob schedule |
+| cleanupJobs.admissionReports.threshold | int | `10000` | Reports threshold, if number of reports are above this value the cronjob will start deleting them |
+| cleanupJobs.admissionReports.history | object | `{"failure":1,"success":1}` | Cronjob history |
+| cleanupJobs.admissionReports.podSecurityContext | object | `{}` | Security context for the pod |
+| cleanupJobs.admissionReports.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}` | Security context for the containers |
+| cleanupJobs.clusterAdmissionReports.enabled | bool | `true` | Enable cleanup cronjob |
+| cleanupJobs.clusterAdmissionReports.image.registry | string | `nil` | Image registry |
+| cleanupJobs.clusterAdmissionReports.image.repository | string | `"bitnami/kubectl"` | Image repository |
+| cleanupJobs.clusterAdmissionReports.image.tag | string | `"1.26.4"` | Image tag Defaults to `latest` if omitted |
+| cleanupJobs.clusterAdmissionReports.image.pullPolicy | string | `nil` | Image pull policy Defaults to image.pullPolicy if omitted |
+| cleanupJobs.clusterAdmissionReports.schedule | string | `"*/10 * * * *"` | Cronjob schedule |
+| cleanupJobs.clusterAdmissionReports.threshold | int | `10000` | Reports threshold, if number of reports are above this value the cronjob will start deleting them |
+| cleanupJobs.clusterAdmissionReports.history | object | `{"failure":1,"success":1}` | Cronjob history |
+| cleanupJobs.clusterAdmissionReports.podSecurityContext | object | `{}` | Security context for the pod |
+| cleanupJobs.clusterAdmissionReports.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}` | Security context for the containers |
+
 ### Other
 
 | Key | Type | Default | Description |

charts/kyverno/README.md.gotmpl

@@ -206,6 +206,7 @@ The chart values are organised per component.
 {{- $test := list -}}
 {{- $apiVersionOverride := list -}}
 {{- $webhooksCleanup := list -}}
+{{- $cleanupJobs := list -}}
 {{- range .Values -}}
   {{- if (hasPrefix "admissionController." .Key) -}}
     {{- $admissionController = append $admissionController . -}}
@@ -231,6 +232,8 @@ The chart values are organised per component.
     {{- $test = append $test . -}}
   {{- else if (hasPrefix "apiVersionOverride." .Key) -}}
     {{- $apiVersionOverride = append $apiVersionOverride . -}}
+  {{- else if (hasPrefix "cleanupJobs." .Key) -}}
+    {{- $cleanupJobs = append $cleanupJobs . -}}
   {{- else -}}
     {{- $other = append $other . -}}
   {{- end -}}
@@ -284,6 +287,10 @@ The chart values are organised per component.
 
 {{ template "chart.valuesTable" (dict "Values" $apiVersionOverride) }}
 
+### Cleanup jobs
+
+{{ template "chart.valuesTable" (dict "Values" $cleanupJobs) }}
+
 ### Other
 
 {{ template "chart.valuesTable" (dict "Values" $other) }}

charts/kyverno/templates/cleanup/cleanup-admission-reports.yaml

@@ -0,0 +1,43 @@
+{{- if .Values.cleanupJobs.admissionReports.enabled -}}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ template "kyverno.name" . }}-cleanup-admission-reports
+  namespace: {{ template "kyverno.namespace" . }}
+  labels:
+    {{- include "kyverno.labels.merge" (list (include "kyverno.labels.common" .) (include "kyverno.matchLabels.common" .)) | nindent 4 }}
+spec:
+  schedule: {{ .Values.cleanupJobs.admissionReports.schedule | quote }}
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: {{ .Values.cleanupJobs.admissionReports.history.success }}
+  failedJobsHistoryLimit: {{ .Values.cleanupJobs.admissionReports.history.failure }}
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: {{ template "kyverno.name" . }}-cleanup-jobs
+          {{- with .Values.cleanupJobs.admissionReports.podSecurityContext }}
+          securityContext:
+            {{- tpl (toYaml .) $ | nindent 12 }}
+          {{- end }}
+          containers:
+          - name: cleanup
+            image: {{ template "kyverno.image" .Values.cleanupJobs.admissionReports }}
+            imagePullPolicy: {{ .Values.cleanupJobs.admissionReports.image.pullPolicy }}
+            command:
+            - /bin/sh
+            - -c
+            - |
+              COUNT=$(kubectl get admissionreports.kyverno.io -A | wc -l)
+              if [ "$COUNT" -gt {{ .Values.cleanupJobs.admissionReports.threshold }} ]; then
+                echo "too many reports found ($COUNT), cleaning up..."
+                kubectl delete admissionreports.kyverno.io -A -l='!audit.kyverno.io/report.aggregate'
+              else
+                echo "($COUNT) reports found, no clean up needed"
+              fi
+            {{- with .Values.cleanupJobs.admissionReports.securityContext }}
+            securityContext:
+              {{- toYaml . | nindent 14 }}
+            {{- end }}
+          restartPolicy: OnFailure
+{{- end -}}

charts/kyverno/templates/cleanup/cleanup-cluster-admission-reports.yaml

@@ -0,0 +1,43 @@
+{{- if .Values.cleanupJobs.clusterAdmissionReports.enabled -}}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ template "kyverno.name" . }}-cleanup-cluster-admission-reports
+  namespace: {{ template "kyverno.namespace" . }}
+  labels:
+    {{- include "kyverno.labels.merge" (list (include "kyverno.labels.common" .) (include "kyverno.matchLabels.common" .)) | nindent 4 }}
+spec:
+  schedule: {{ .Values.cleanupJobs.clusterAdmissionReports.schedule | quote }}
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: {{ .Values.cleanupJobs.clusterAdmissionReports.history.success }}
+  failedJobsHistoryLimit: {{ .Values.cleanupJobs.clusterAdmissionReports.history.failure }}
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: {{ template "kyverno.name" . }}-cleanup-jobs
+          {{- with .Values.cleanupJobs.clusterAdmissionReports.podSecurityContext }}
+          securityContext:
+            {{- tpl (toYaml .) $ | nindent 12 }}
+          {{- end }}
+          containers:
+          - name: cleanup
+            image: {{ template "kyverno.image" .Values.cleanupJobs.clusterAdmissionReports }}
+            imagePullPolicy: {{ .Values.cleanupJobs.clusterAdmissionReports.image.pullPolicy }}
+            command:
+            - /bin/sh
+            - -c
+            - |
+              COUNT=$(kubectl get clusteradmissionreports.kyverno.io -A | wc -l)
+              if [ "$COUNT" -gt {{ .Values.cleanupJobs.clusterAdmissionReports.threshold }} ]; then
+                echo "too many reports found ($COUNT), cleaning up..."
+                kubectl delete clusteradmissionreports.kyverno.io -A -l='!audit.kyverno.io/report.aggregate'
+              else
+                echo "($COUNT) reports found, no clean up needed"
+              fi
+            {{- with .Values.cleanupJobs.clusterAdmissionReports.securityContext }}
+            securityContext:
+              {{- toYaml . | nindent 14 }}
+            {{- end }}
+          restartPolicy: OnFailure
+{{- end -}}

charts/kyverno/templates/cleanup/clusterrole.yaml

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ template "kyverno.name" . }}-cleanup-jobs
+  labels:
+    {{- include "kyverno.labels.merge" (list (include "kyverno.labels.common" .) (include "kyverno.matchLabels.common" .)) | nindent 4 }}
+rules:
+  - apiGroups:
+      - kyverno.io
+    resources:
+      - admissionreports
+      - clusteradmissionreports
+    verbs:
+      - list
+      - deletecollection

charts/kyverno/templates/cleanup/clusterrolebinding.yaml

@@ -0,0 +1,14 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "kyverno.name" . }}-cleanup-jobs
+  labels:
+    {{- include "kyverno.labels.merge" (list (include "kyverno.labels.common" .) (include "kyverno.matchLabels.common" .)) | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ template "kyverno.name" . }}-cleanup-jobs
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "kyverno.name" . }}-cleanup-jobs
+    namespace: {{ template "kyverno.namespace" . }}

charts/kyverno/templates/cleanup/serviceaccount.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "kyverno.name" . }}-cleanup-jobs
+  namespace: {{ template "kyverno.namespace" . }}
+  labels:
+    {{- include "kyverno.labels.merge" (list (include "kyverno.labels.common" .) (include "kyverno.matchLabels.common" .)) | nindent 4 }}

charts/kyverno/values.yaml

@@ -357,6 +357,95 @@ features:
     # -- Reports chunk size
     chunkSize: 1000
 
+# Cleanup cronjobs to prevent internal resources from stacking up in the cluster
+cleanupJobs:
+
+  admissionReports:
+
+    # -- Enable cleanup cronjob
+    enabled: true
+
+    image:
+      # -- (string) Image registry
+      registry: ~
+      # -- Image repository
+      repository: bitnami/kubectl
+      # -- Image tag
+      # Defaults to `latest` if omitted
+      tag: '1.26.4'
+      # -- (string) Image pull policy
+      # Defaults to image.pullPolicy if omitted
+      pullPolicy: ~
+
+    # -- Cronjob schedule
+    schedule: '*/10 * * * *'
+
+    # -- Reports threshold, if number of reports are above this value the cronjob will start deleting them
+    threshold: 10000
+
+    # -- Cronjob history
+    history:
+      success: 1
+      failure: 1
+
+    # -- Security context for the pod
+    podSecurityContext: {}
+
+    # -- Security context for the containers
+    securityContext:
+      runAsNonRoot: true
+      privileged: false
+      allowPrivilegeEscalation: false
+      readOnlyRootFilesystem: true
+      capabilities:
+        drop:
+          - ALL
+      seccompProfile:
+        type: RuntimeDefault
+
+  clusterAdmissionReports:
+
+    # -- Enable cleanup cronjob
+    enabled: true
+
+    image:
+      # -- (string) Image registry
+      registry: ~
+      # -- Image repository
+      repository: bitnami/kubectl
+      # -- Image tag
+      # Defaults to `latest` if omitted
+      tag: '1.26.4'
+      # -- (string) Image pull policy
+      # Defaults to image.pullPolicy if omitted
+      pullPolicy: ~
+
+    # -- Cronjob schedule
+    schedule: '*/10 * * * *'
+
+    # -- Reports threshold, if number of reports are above this value the cronjob will start deleting them
+    threshold: 10000
+
+    # -- Cronjob history
+    history:
+      success: 1
+      failure: 1
+
+    # -- Security context for the pod
+    podSecurityContext: {}
+
+    # -- Security context for the containers
+    securityContext:
+      runAsNonRoot: true
+      privileged: false
+      allowPrivilegeEscalation: false
+      readOnlyRootFilesystem: true
+      capabilities:
+        drop:
+          - ALL
+      seccompProfile:
+        type: RuntimeDefault
+
 # Admission controller configuration
 admissionController:
 

config/install-latest-testing.yaml

@@ -43,6 +43,16 @@ metadata:
 ---
 apiVersion: v1
 kind: ServiceAccount
+metadata:
+  name: kyverno-cleanup-jobs
+  namespace: kyverno
+  labels:
+    app.kubernetes.io/instance: kyverno
+    app.kubernetes.io/part-of: kyverno
+    app.kubernetes.io/version: latest
+---
+apiVersion: v1
+kind: ServiceAccount
 metadata:
   name: kyverno-reports-controller
   namespace: kyverno
@@ -34158,6 +34168,24 @@ rules:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
+metadata:
+  name: kyverno-cleanup-jobs
+  labels:
+    app.kubernetes.io/instance: kyverno
+    app.kubernetes.io/part-of: kyverno
+    app.kubernetes.io/version: latest
+rules:
+  - apiGroups:
+      - kyverno.io
+    resources:
+      - admissionreports
+      - clusteradmissionreports
+    verbs:
+      - list
+      - deletecollection
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
   name: kyverno:rbac:admin:policies
   labels:
@@ -34477,6 +34505,23 @@ subjects:
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kyverno-cleanup-jobs
+  labels:
+    app.kubernetes.io/instance: kyverno
+    app.kubernetes.io/part-of: kyverno
+    app.kubernetes.io/version: latest
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kyverno-cleanup-jobs
+subjects:
+  - kind: ServiceAccount
+    name: kyverno-cleanup-jobs
+    namespace: kyverno
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: kyverno:reports-controller
   labels:
@@ -35357,3 +35402,95 @@ spec:
       volumes:
       - name: sigstore
         emptyDir: {}
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: kyverno-cleanup-admission-reports
+  namespace: kyverno
+  labels:
+    app.kubernetes.io/instance: kyverno
+    app.kubernetes.io/part-of: kyverno
+    app.kubernetes.io/version: latest
+spec:
+  schedule: "*/10 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: kyverno-cleanup-jobs
+          containers:
+          - name: cleanup
+            image: bitnami/kubectl:1.26.4
+            imagePullPolicy: 
+            command:
+            - /bin/sh
+            - -c
+            - |
+              COUNT=$(kubectl get admissionreports.kyverno.io -A | wc -l)
+              if [ "$COUNT" -gt 10000 ]; then
+                echo "too many reports found ($COUNT), cleaning up..."
+                kubectl delete admissionreports.kyverno.io -A -l='!audit.kyverno.io/report.aggregate'
+              else
+                echo "($COUNT) reports found, no clean up needed"
+              fi
+            securityContext:
+              allowPrivilegeEscalation: false
+              capabilities:
+                drop:
+                - ALL
+              privileged: false
+              readOnlyRootFilesystem: true
+              runAsNonRoot: true
+              seccompProfile:
+                type: RuntimeDefault
+          restartPolicy: OnFailure
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: kyverno-cleanup-cluster-admission-reports
+  namespace: kyverno
+  labels:
+    app.kubernetes.io/instance: kyverno
+    app.kubernetes.io/part-of: kyverno
+    app.kubernetes.io/version: latest
+spec:
+  schedule: "*/10 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: kyverno-cleanup-jobs
+          containers:
+          - name: cleanup
+            image: bitnami/kubectl:1.26.4
+            imagePullPolicy: 
+            command:
+            - /bin/sh
+            - -c
+            - |
+              COUNT=$(kubectl get clusteradmissionreports.kyverno.io -A | wc -l)
+              if [ "$COUNT" -gt 10000 ]; then
+                echo "too many reports found ($COUNT), cleaning up..."
+                kubectl delete clusteradmissionreports.kyverno.io -A -l='!audit.kyverno.io/report.aggregate'
+              else
+                echo "($COUNT) reports found, no clean up needed"
+              fi
+            securityContext:
+              allowPrivilegeEscalation: false
+              capabilities:
+                drop:
+                - ALL
+              privileged: false
+              readOnlyRootFilesystem: true
+              runAsNonRoot: true
+              seccompProfile:
+                type: RuntimeDefault
+          restartPolicy: OnFailure