refactor: helm admission controller config (#6460)

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

charts/kyverno/README.md

@@ -139,6 +139,10 @@ In `v3` chart values changed significantly, please read the instructions below t
 - `livenessProbe` has been replaced with `admissionController.livenessProbe`
 - `readinessProbe` has been replaced with `admissionController.readinessProbe`
 - `createSelfSignedCert` has been replaced with `admissionController.createSelfSignedCert`
+- `serviceMonitor` has been replaced with `admissionController.serviceMonitor`
+- `podSecurityContext` has been replaced with `admissionController.podSecurityContext`
+- `tufRootMountPath` has been replaced with `admissionController.tufRootMountPath`
+- `sigstoreVolume` has been replaced with `admissionController.sigstoreVolume`
 
 - Labels and selectors have been reworked and due to immutability, upgrading from `v2` to `v3` is going to be rejected. The easiest solution is to uninstall `v2` and reinstall `v3` once values have been adapted to the changes described above.
 
@@ -207,7 +211,6 @@ The command removes all the Kubernetes components associated with the chart and
 | initContainer.extraArgs | list | `["--loggingFormat=text"]` | Extra arguments to give to the kyvernopre binary. |
 | podLabels | object | `{}` | Additional labels to add to each pod |
 | podAnnotations | object | `{}` | Additional annotations to add to each pod |
-| podSecurityContext | object | `{}` | Security context for the pod |
 | securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}` | Security context for the containers |
 | envVarsInit | object | `{}` | Env variables for initContainers. |
 | envVars | object | `{}` | Env variables for containers. |
@@ -230,19 +233,10 @@ The command removes all the Kubernetes components associated with the chart and
 | metricsService.type | string | `"ClusterIP"` | Service type. |
 | metricsService.nodePort | string | `nil` | Service node port. Only used if `metricsService.type` is `NodePort`. |
 | metricsService.annotations | object | `{}` | Service annotations. |
-| serviceMonitor.enabled | bool | `false` | Create a `ServiceMonitor` to collect Prometheus metrics. |
-| serviceMonitor.additionalLabels | string | `nil` | Additional labels |
-| serviceMonitor.namespace | string | `nil` | Override namespace (default is the same as kyverno) |
-| serviceMonitor.interval | string | `"30s"` | Interval to scrape metrics |
-| serviceMonitor.scrapeTimeout | string | `"25s"` | Timeout if metrics can't be retrieved in given time interval |
-| serviceMonitor.secure | bool | `false` | Is TLS required for endpoint |
-| serviceMonitor.tlsConfig | object | `{}` | TLS Configuration for endpoint |
 | networkPolicy.enabled | bool | `false` | When true, use a NetworkPolicy to allow ingress to the webhook This is useful on clusters using Calico and/or native k8s network policies in a default-deny setup. |
 | networkPolicy.ingressFrom | list | `[]` | A list of valid from selectors according to https://kubernetes.io/docs/concepts/services-networking/network-policies. |
 | webhooksCleanup.enabled | bool | `false` | Create a helm pre-delete hook to cleanup webhooks. |
 | webhooksCleanup.image | string | `"bitnami/kubectl:latest"` | `kubectl` image to run commands for deleting webhooks. |
-| tufRootMountPath | string | `"/.sigstore"` | A writable volume to use for the TUF root initialization. |
-| sigstoreVolume | object | `{"emptyDir":{}}` | Volume to be mounted in pods for TUF/cosign work. |
 | grafana.enabled | bool | `false` | Enable grafana dashboard creation. |
 | grafana.configMapName | string | `"{{ include \"kyverno.fullname\" . }}-grafana"` | Configmap name template. |
 | grafana.namespace | string | `nil` | Namespace to create the grafana dashboard configmap. If not set, it will be created in the same namespace where the chart is deployed. |
@@ -263,8 +257,18 @@ The command removes all the Kubernetes components associated with the chart and
 | admissionController.podAffinity | object | `{}` | Pod affinity constraints. |
 | admissionController.nodeAffinity | object | `{}` | Node affinity constraints. |
 | admissionController.topologySpreadConstraints | list | `[]` | Topology spread constraints. |
+| admissionController.podSecurityContext | object | `{}` | Security context for the pod |
 | admissionController.podDisruptionBudget.minAvailable | int | `1` | Configures the minimum available pods for disruptions. Cannot be used if `maxUnavailable` is set. |
 | admissionController.podDisruptionBudget.maxUnavailable | string | `nil` | Configures the maximum unavailable pods for disruptions. Cannot be used if `minAvailable` is set. |
+| admissionController.serviceMonitor.enabled | bool | `false` | Create a `ServiceMonitor` to collect Prometheus metrics. |
+| admissionController.serviceMonitor.additionalLabels | object | `{}` | Additional labels |
+| admissionController.serviceMonitor.namespace | string | `nil` | Override namespace |
+| admissionController.serviceMonitor.interval | string | `"30s"` | Interval to scrape metrics |
+| admissionController.serviceMonitor.scrapeTimeout | string | `"25s"` | Timeout if metrics can't be retrieved in given time interval |
+| admissionController.serviceMonitor.secure | bool | `false` | Is TLS required for endpoint |
+| admissionController.serviceMonitor.tlsConfig | object | `{}` | TLS Configuration for endpoint |
+| admissionController.tufRootMountPath | string | `"/.sigstore"` | A writable volume to use for the TUF root initialization. |
+| admissionController.sigstoreVolume | object | `{"emptyDir":{}}` | Volume to be mounted in pods for TUF/cosign work. |
 | cleanupController.enabled | bool | `true` | Enable cleanup controller. |
 | cleanupController.rbac.create | bool | `true` | Create RBAC resources |
 | cleanupController.rbac.serviceAccount.name | string | `nil` | Service account name |
@@ -307,8 +311,8 @@ The command removes all the Kubernetes components associated with the chart and
 | cleanupController.metricsService.nodePort | string | `nil` | Service node port. Only used if `metricsService.type` is `NodePort`. |
 | cleanupController.metricsService.annotations | object | `{}` | Service annotations. |
 | cleanupController.serviceMonitor.enabled | bool | `false` | Create a `ServiceMonitor` to collect Prometheus metrics. |
-| cleanupController.serviceMonitor.additionalLabels | string | `nil` | Additional labels |
-| cleanupController.serviceMonitor.namespace | string | `nil` | Override namespace (default is the same as kyverno) |
+| cleanupController.serviceMonitor.additionalLabels | object | `{}` | Additional labels |
+| cleanupController.serviceMonitor.namespace | string | `nil` | Override namespace |
 | cleanupController.serviceMonitor.interval | string | `"30s"` | Interval to scrape metrics |
 | cleanupController.serviceMonitor.scrapeTimeout | string | `"25s"` | Timeout if metrics can't be retrieved in given time interval |
 | cleanupController.serviceMonitor.secure | bool | `false` | Is TLS required for endpoint |
@@ -357,8 +361,8 @@ The command removes all the Kubernetes components associated with the chart and
 | reportsController.metricsService.nodePort | string | `nil` | Service node port. Only used if `metricsService.type` is `NodePort`. |
 | reportsController.metricsService.annotations | object | `{}` | Service annotations. |
 | reportsController.serviceMonitor.enabled | bool | `false` | Create a `ServiceMonitor` to collect Prometheus metrics. |
-| reportsController.serviceMonitor.additionalLabels | string | `nil` | Additional labels |
-| reportsController.serviceMonitor.namespace | string | `nil` | Override namespace (default is the same as kyverno) |
+| reportsController.serviceMonitor.additionalLabels | object | `{}` | Additional labels |
+| reportsController.serviceMonitor.namespace | string | `nil` | Override namespace |
 | reportsController.serviceMonitor.interval | string | `"30s"` | Interval to scrape metrics |
 | reportsController.serviceMonitor.scrapeTimeout | string | `"25s"` | Timeout if metrics can't be retrieved in given time interval |
 | reportsController.serviceMonitor.secure | bool | `false` | Is TLS required for endpoint |
@@ -407,8 +411,8 @@ The command removes all the Kubernetes components associated with the chart and
 | backgroundController.metricsService.nodePort | string | `nil` | Service node port. Only used if `metricsService.type` is `NodePort`. |
 | backgroundController.metricsService.annotations | object | `{}` | Service annotations. |
 | backgroundController.serviceMonitor.enabled | bool | `false` | Create a `ServiceMonitor` to collect Prometheus metrics. |
-| backgroundController.serviceMonitor.additionalLabels | string | `nil` | Additional labels |
-| backgroundController.serviceMonitor.namespace | string | `nil` | Override namespace (default is the same as kyverno) |
+| backgroundController.serviceMonitor.additionalLabels | object | `{}` | Additional labels |
+| backgroundController.serviceMonitor.namespace | string | `nil` | Override namespace |
 | backgroundController.serviceMonitor.interval | string | `"30s"` | Interval to scrape metrics |
 | backgroundController.serviceMonitor.scrapeTimeout | string | `"25s"` | Timeout if metrics can't be retrieved in given time interval |
 | backgroundController.serviceMonitor.secure | bool | `false` | Is TLS required for endpoint |

charts/kyverno/README.md.gotmpl

@@ -139,6 +139,10 @@ In `v3` chart values changed significantly, please read the instructions below t
 - `livenessProbe` has been replaced with `admissionController.livenessProbe`
 - `readinessProbe` has been replaced with `admissionController.readinessProbe`
 - `createSelfSignedCert` has been replaced with `admissionController.createSelfSignedCert`
+- `serviceMonitor` has been replaced with `admissionController.serviceMonitor`
+- `podSecurityContext` has been replaced with `admissionController.podSecurityContext`
+- `tufRootMountPath` has been replaced with `admissionController.tufRootMountPath`
+- `sigstoreVolume` has been replaced with `admissionController.sigstoreVolume`
 
 - Labels and selectors have been reworked and due to immutability, upgrading from `v2` to `v3` is going to be rejected. The easiest solution is to uninstall `v2` and reinstall `v3` once values have been adapted to the changes described above.
 

charts/kyverno/templates/admission-controller/deployment.yaml

@@ -30,7 +30,7 @@ spec:
       imagePullSecrets:
         {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
-      {{- with .Values.podSecurityContext }}
+      {{- with .Values.admissionController.podSecurityContext }}
       securityContext:
         {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
@@ -156,7 +156,7 @@ spec:
           - name: KYVERNO_SVC
             value: {{ template "kyverno.admission-controller.serviceName" . }}
           - name: TUF_ROOT
-            value: {{ .Values.tufRootMountPath }}
+            value: {{ .Values.admissionController.tufRootMountPath }}
           {{- with .Values.envVars }}
           {{- toYaml . | nindent 10 }}
           {{- end }}
@@ -175,13 +175,13 @@ spec:
             {{- tpl (toYaml .) $ | nindent 12 }}
           {{- end }}
           volumeMounts:
-            - mountPath: {{ .Values.tufRootMountPath }}
+            - mountPath: {{ .Values.admissionController.tufRootMountPath }}
               name: sigstore
             - mountPath: /var/run/secrets/tokens
               name: api-token
       volumes:
       - name: sigstore
-        {{- toYaml (required "A valid .Values.sigstoreVolume entry is required" .Values.sigstoreVolume) | nindent 8 }}
+        {{- toYaml (required "A valid .Values.admissionController.sigstoreVolume entry is required" .Values.admissionController.sigstoreVolume) | nindent 8 }}
       - name: api-token
         projected:
           sources:

charts/kyverno/templates/admission-controller/servicemonitor.yaml

@@ -1,17 +1,17 @@
-{{- if .Values.serviceMonitor.enabled }}
-apiVersion: "monitoring.coreos.com/v1"
+{{- if .Values.admissionController.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   name: {{ template "kyverno.admission-controller.name" . }}
-  {{- if .Values.serviceMonitor.namespace }}
-  namespace: {{ .Values.serviceMonitor.namespace }}
+  {{- if .Values.admissionController.serviceMonitor.namespace }}
+  namespace: {{ .Values.admissionController.serviceMonitor.namespace }}
   {{- else }}
   namespace: {{ template "kyverno.namespace" . }}
   {{- end }}
   labels:
     {{- include "kyverno.admission-controller.labels" . | nindent 4 }}
-    {{- if .Values.serviceMonitor.additionalLabels }}
-    {{- toYaml .Values.serviceMonitor.additionalLabels | nindent 4 }}
+    {{- with .Values.admissionController.serviceMonitor.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
     {{- end }}
 spec:
   selector:
@@ -22,11 +22,11 @@ spec:
     - {{ template "kyverno.namespace" . }}
   endpoints:
   - port: metrics-port
-    interval: {{ .Values.serviceMonitor.interval }}
-    scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }}
-  {{- if .Values.serviceMonitor.secure }}
+    interval: {{ .Values.admissionController.serviceMonitor.interval }}
+    scrapeTimeout: {{ .Values.admissionController.serviceMonitor.scrapeTimeout }}
+    {{- if .Values.admissionController.serviceMonitor.secure }}
     scheme: https
     tlsConfig:
-      {{- toYaml .Values.serviceMonitor.tlsConfig | nindent 8 }}
+      {{- toYaml .Values.admissionController.serviceMonitor.tlsConfig | nindent 8 }}
     {{- end }}
-{{- end }}
+{{- end -}}

charts/kyverno/values.yaml

@@ -87,7 +87,7 @@ config:
     - '[Secret,{{ include "kyverno.namespace" . }},{{ template "kyverno.admission-controller.serviceName" . }}.{{ template "kyverno.namespace" . }}.svc.*]'
     - '[Service,{{ include "kyverno.namespace" . }},{{ template "kyverno.admission-controller.serviceName" . }}]'
     - '[Service,{{ include "kyverno.namespace" . }},{{ template "kyverno.admission-controller.serviceName" . }}-metrics]'
-    - '[ServiceMonitor,{{ if .Values.serviceMonitor.namespace }}{{ .Values.serviceMonitor.namespace }}{{ else }}{{ template "kyverno.namespace" . }}{{ end }},{{ template "kyverno.admission-controller.serviceName" . }}-service-monitor]'
+    - '[ServiceMonitor,{{ if .Values.admissionController.serviceMonitor.namespace }}{{ .Values.admissionController.serviceMonitor.namespace }}{{ else }}{{ template "kyverno.namespace" . }}{{ end }},{{ template "kyverno.admission-controller.serviceName" . }}-service-monitor]'
     - '[Pod,{{ include "kyverno.namespace" . }},{{ template "kyverno.fullname" . }}-*]'
 
   # -- Defines the `namespaceSelector` in the webhook configurations.
@@ -242,9 +242,6 @@ podLabels: {}
 podAnnotations: {}
   # example.com/annotation: foo
 
-# -- Security context for the pod
-podSecurityContext: {}
-
 # -- Security context for the containers
 securityContext:
   runAsNonRoot: true
@@ -338,23 +335,6 @@ metricsService:
   # -- Service annotations.
   annotations: {}
 
-serviceMonitor:
-  # -- Create a `ServiceMonitor` to collect Prometheus metrics.
-  enabled: false
-  # -- Additional labels
-  additionalLabels:
-    # key: value
-  # -- Override namespace (default is the same as kyverno)
-  namespace:
-  # --  Interval to scrape metrics
-  interval: 30s
-  # -- Timeout if metrics can't be retrieved in given time interval
-  scrapeTimeout: 25s
-  # -- Is TLS required for endpoint
-  secure: false
-  # -- TLS Configuration for endpoint
-  tlsConfig: {}
-
 networkPolicy:
   # -- When true, use a NetworkPolicy to allow ingress to the webhook
   # This is useful on clusters using Calico and/or native k8s network policies in a default-deny setup.
@@ -368,13 +348,6 @@ webhooksCleanup:
   # -- `kubectl` image to run commands for deleting webhooks.
   image: bitnami/kubectl:latest
 
-# -- A writable volume to use for the TUF root initialization.
-tufRootMountPath: /.sigstore
-
-# -- Volume to be mounted in pods for TUF/cosign work.
-sigstoreVolume:
-  emptyDir: {}
-
 grafana:
   # -- Enable grafana dashboard creation.
   enabled: false
@@ -498,6 +471,9 @@ admissionController:
   # -- Topology spread constraints.
   topologySpreadConstraints: []
 
+  # -- Security context for the pod
+  podSecurityContext: {}
+
   podDisruptionBudget:
     # -- Configures the minimum available pods for disruptions.
     # Cannot be used if `maxUnavailable` is set.
@@ -506,6 +482,29 @@ admissionController:
     # Cannot be used if `minAvailable` is set.
     maxUnavailable:
 
+  serviceMonitor:
+    # -- Create a `ServiceMonitor` to collect Prometheus metrics.
+    enabled: false
+    # -- Additional labels
+    additionalLabels: {}
+    # -- (string) Override namespace
+    namespace: ~
+    # --  Interval to scrape metrics
+    interval: 30s
+    # -- Timeout if metrics can't be retrieved in given time interval
+    scrapeTimeout: 25s
+    # -- Is TLS required for endpoint
+    secure: false
+    # -- TLS Configuration for endpoint
+    tlsConfig: {}
+
+  # -- A writable volume to use for the TUF root initialization.
+  tufRootMountPath: /.sigstore
+
+  # -- Volume to be mounted in pods for TUF/cosign work.
+  sigstoreVolume:
+    emptyDir: {}
+
 # Cleanup controller configuration
 cleanupController:
 
@@ -712,10 +711,9 @@ cleanupController:
     # -- Create a `ServiceMonitor` to collect Prometheus metrics.
     enabled: false
     # -- Additional labels
-    additionalLabels:
-      # key: value
-    # -- Override namespace (default is the same as kyverno)
-    namespace:
+    additionalLabels: {}
+    # -- (string) Override namespace
+    namespace: ~
     # --  Interval to scrape metrics
     interval: 30s
     # -- Timeout if metrics can't be retrieved in given time interval
@@ -943,10 +941,9 @@ reportsController:
     # -- Create a `ServiceMonitor` to collect Prometheus metrics.
     enabled: false
     # -- Additional labels
-    additionalLabels:
-      # key: value
-    # -- Override namespace (default is the same as kyverno)
-    namespace:
+    additionalLabels: {}
+    # -- (string) Override namespace
+    namespace: ~
     # --  Interval to scrape metrics
     interval: 30s
     # -- Timeout if metrics can't be retrieved in given time interval
@@ -1133,10 +1130,9 @@ backgroundController:
     # -- Create a `ServiceMonitor` to collect Prometheus metrics.
     enabled: false
     # -- Additional labels
-    additionalLabels:
-      # key: value
-    # -- Override namespace (default is the same as kyverno)
-    namespace:
+    additionalLabels: {}
+    # -- (string) Override namespace
+    namespace: ~
     # --  Interval to scrape metrics
     interval: 30s
     # -- Timeout if metrics can't be retrieved in given time interval

scripts/config/dev/kyverno.yaml

@@ -9,12 +9,13 @@ extraArgs:
   - --tracingPort=4317
   - --enablePolicyException
 
-serviceMonitor:
-  enabled: true
-
 grafana:
   enabled: true
 
+admissionController:
+  serviceMonitor:
+    enabled: true
+
 cleanupController:
   rbac:
     clusterRole: