refactor: helm admission controller config (#6474)

* refactor: helm admission controller config

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* services

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

Makefile

@@ -521,7 +521,7 @@ codegen-manifest-install: $(HELM) ## Create install manifest
 	@$(HELM) template kyverno --namespace kyverno --skip-tests ./charts/kyverno \
 		--set templating.enabled=true \
 		--set templating.version=latest \
-		--set image.tag=latest \
+		--set admissionController.container.image.tag=latest \
 		--set admissionController.initContainer.image.tag=latest \
 		--set cleanupController.image.tag=latest \
 		--set reportsController.image.tag=latest \
@@ -537,7 +537,7 @@ codegen-manifest-debug: $(HELM) ## Create debug manifest
 		--set templating.enabled=true \
 		--set templating.version=latest \
 		--set templating.debug=true \
-		--set image.tag=latest \
+		--set admissionController.container.image.tag=latest \
 		--set admissionController.initContainer.image.tag=latest \
 		--set cleanupController.image.tag=latest \
 		--set reportsController.image.tag=latest \
@@ -552,7 +552,7 @@ codegen-manifest-release: $(HELM) ## Create release manifest
 	@$(HELM) template kyverno --namespace kyverno --skip-tests ./charts/kyverno \
 		--set templating.enabled=true \
 		--set templating.version=$(GIT_VERSION) \
-		--set image.tag=$(GIT_VERSION) \
+		--set admissionController.container.image.tag=$(GIT_VERSION) \
 		--set admissionController.initContainer.image.tag=$(GIT_VERSION) \
 		--set cleanupController.image.tag=$(GIT_VERSION) \
 		--set reportsController.image.tag=$(GIT_VERSION) \
@@ -816,9 +816,9 @@ kind-load-image-archive: $(KIND) ## Load docker images from archive
 kind-install-kyverno: $(HELM) ## Install kyverno helm chart
 	@echo Install kyverno chart... >&2
 	@$(HELM) upgrade --install kyverno --namespace kyverno --create-namespace --wait ./charts/kyverno \
-		--set image.registry=$(LOCAL_REGISTRY) \
-		--set image.repository=$(LOCAL_KYVERNO_REPO) \
-		--set image.tag=$(IMAGE_TAG_DEV) \
+		--set admissionController.container.image.registry=$(LOCAL_REGISTRY) \
+		--set admissionController.container.image.repository=$(LOCAL_KYVERNO_REPO) \
+		--set admissionController.container.image.tag=$(IMAGE_TAG_DEV) \
 		--set admissionController.initContainer.image.registry=$(LOCAL_REGISTRY) \
 		--set admissionController.initContainer.image.repository=$(LOCAL_KYVERNOPRE_REPO) \
 		--set admissionController.initContainer.image.tag=$(IMAGE_TAG_DEV) \

charts/kyverno/README.md

@@ -145,6 +145,11 @@ In `v3` chart values changed significantly, please read the instructions below t
 - `sigstoreVolume` has been replaced with `admissionController.sigstoreVolume`
 - `initImage` has been replaced with `admissionController.initContainer.image`
 - `initResources` has been replaced with `admissionController.initContainer.resources`
+- `image` has been replaced with `admissionController.container.image`
+- `image.pullSecrets` has been replaced with `admissionController.pullSecrets`
+- `resources` has been replaced with `admissionController.container.resources`
+- `service` has been replaced with `admissionController.service`
+- `metricsService` has been replaced with `admissionController.metricsService`
 
 - Labels and selectors have been reworked and due to immutability, upgrading from `v2` to `v3` is going to be rejected. The easiest solution is to uninstall `v2` and reinstall `v3` once values have been adapted to the changes described above.
 
@@ -201,11 +206,6 @@ The command removes all the Kubernetes components associated with the chart and
 | rbac.serviceAccount.create | bool | `true` | Create a ServiceAccount |
 | rbac.serviceAccount.name | string | `nil` | The ServiceAccount name |
 | rbac.serviceAccount.annotations | object | `{}` | Annotations for the ServiceAccount |
-| image.registry | string | `"ghcr.io"` | Image registry |
-| image.repository | string | `"kyverno/kyverno"` | Image repository |
-| image.tag | string | `nil` | Image tag Defaults to appVersion in Chart.yaml if omitted |
-| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
-| image.pullSecrets | list | `[]` | Image pull secrets |
 | initContainer.extraArgs | list | `["--loggingFormat=text"]` | Extra arguments to give to the kyvernopre binary. |
 | podLabels | object | `{}` | Additional labels to add to each pod |
 | podAnnotations | object | `{}` | Additional annotations to add to each pod |
@@ -215,20 +215,9 @@ The command removes all the Kubernetes components associated with the chart and
 | extraArgs | list | `["--loggingFormat=text"]` | Extra arguments to give to the binary. |
 | extraInitContainers | list | `[]` | Array of extra init containers |
 | extraContainers | list | `[]` | Array of extra containers to run alongside kyverno |
-| resources.limits | object | `{"memory":"384Mi"}` | Pod resource limits |
-| resources.requests | object | `{"cpu":"100m","memory":"128Mi"}` | Pod resource requests |
 | generatecontrollerExtraResources | list | `[]` | Additional resources to be added to controller RBAC permissions. |
 | excludeKyvernoNamespace | bool | `true` | Exclude Kyverno namespace Determines if default Kyverno namespace exclusion is enabled for webhooks and resourceFilters |
 | resourceFiltersExcludeNamespaces | list | `[]` | resourceFilter namespace exclude Namespaces to exclude from the default resourceFilters |
-| service.port | int | `443` | Service port. |
-| service.type | string | `"ClusterIP"` | Service type. |
-| service.nodePort | string | `nil` | Service node port. Only used if `service.type` is `NodePort`. |
-| service.annotations | object | `{}` | Service annotations. |
-| metricsService.create | bool | `true` | Create service. |
-| metricsService.port | int | `8000` | Service port. Kyverno's metrics server will be exposed at this port. |
-| metricsService.type | string | `"ClusterIP"` | Service type. |
-| metricsService.nodePort | string | `nil` | Service node port. Only used if `metricsService.type` is `NodePort`. |
-| metricsService.annotations | object | `{}` | Service annotations. |
 | networkPolicy.enabled | bool | `false` | When true, use a NetworkPolicy to allow ingress to the webhook This is useful on clusters using Calico and/or native k8s network policies in a default-deny setup. |
 | networkPolicy.ingressFrom | list | `[]` | A list of valid from selectors according to https://kubernetes.io/docs/concepts/services-networking/network-policies. |
 | webhooksCleanup.enabled | bool | `false` | Create a helm pre-delete hook to cleanup webhooks. |
@@ -265,12 +254,28 @@ The command removes all the Kubernetes components associated with the chart and
 | admissionController.serviceMonitor.tlsConfig | object | `{}` | TLS Configuration for endpoint |
 | admissionController.tufRootMountPath | string | `"/.sigstore"` | A writable volume to use for the TUF root initialization. |
 | admissionController.sigstoreVolume | object | `{"emptyDir":{}}` | Volume to be mounted in pods for TUF/cosign work. |
+| admissionController.pullSecrets | list | `[]` | Image pull secrets |
 | admissionController.initContainer.image.registry | string | `"ghcr.io"` | Image registry |
 | admissionController.initContainer.image.repository | string | `"kyverno/kyvernopre"` | Image repository |
 | admissionController.initContainer.image.tag | string | `nil` | Image tag If missing, defaults to image.tag |
 | admissionController.initContainer.image.pullPolicy | string | `nil` | Image pull policy If missing, defaults to image.pullPolicy |
 | admissionController.initContainer.resources.limits | object | `{"cpu":"100m","memory":"256Mi"}` | Pod resource limits |
 | admissionController.initContainer.resources.requests | object | `{"cpu":"10m","memory":"64Mi"}` | Pod resource requests |
+| admissionController.container.image.registry | string | `"ghcr.io"` | Image registry |
+| admissionController.container.image.repository | string | `"kyverno/kyverno"` | Image repository |
+| admissionController.container.image.tag | string | `nil` | Image tag Defaults to appVersion in Chart.yaml if omitted |
+| admissionController.container.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
+| admissionController.container.resources.limits | object | `{"memory":"384Mi"}` | Pod resource limits |
+| admissionController.container.resources.requests | object | `{"cpu":"100m","memory":"128Mi"}` | Pod resource requests |
+| admissionController.service.port | int | `443` | Service port. |
+| admissionController.service.type | string | `"ClusterIP"` | Service type. |
+| admissionController.service.nodePort | string | `nil` | Service node port. Only used if `type` is `NodePort`. |
+| admissionController.service.annotations | object | `{}` | Service annotations. |
+| admissionController.metricsService.create | bool | `true` | Create service. |
+| admissionController.metricsService.port | int | `8000` | Service port. Kyverno's metrics server will be exposed at this port. |
+| admissionController.metricsService.type | string | `"ClusterIP"` | Service type. |
+| admissionController.metricsService.nodePort | string | `nil` | Service node port. Only used if `type` is `NodePort`. |
+| admissionController.metricsService.annotations | object | `{}` | Service annotations. |
 | cleanupController.enabled | bool | `true` | Enable cleanup controller. |
 | cleanupController.rbac.create | bool | `true` | Create RBAC resources |
 | cleanupController.rbac.serviceAccount.name | string | `nil` | Service account name |

charts/kyverno/README.md.gotmpl

@@ -145,6 +145,11 @@ In `v3` chart values changed significantly, please read the instructions below t
 - `sigstoreVolume` has been replaced with `admissionController.sigstoreVolume`
 - `initImage` has been replaced with `admissionController.initContainer.image`
 - `initResources` has been replaced with `admissionController.initContainer.resources`
+- `image` has been replaced with `admissionController.container.image`
+- `image.pullSecrets` has been replaced with `admissionController.pullSecrets`
+- `resources` has been replaced with `admissionController.container.resources`
+- `service` has been replaced with `admissionController.service`
+- `metricsService` has been replaced with `admissionController.metricsService`
 
 - Labels and selectors have been reworked and due to immutability, upgrading from `v2` to `v3` is going to be rejected. The easiest solution is to uninstall `v2` and reinstall `v3` once values have been adapted to the changes described above.
 

charts/kyverno/ci/imageRegistry-values.yaml

@@ -3,11 +3,12 @@ test:
     registry: docker.io
     repository: busybox
 
-image:
-  registry: ko.local
-  repository: github.com/kyverno/kyverno/cmd/kyverno
-
 admissionController:
+  container:
+    image:
+      registry: ko.local
+      repository: github.com/kyverno/kyverno/cmd/kyverno
+
   initContainer:
     image:
       registry: ko.local

charts/kyverno/templates/NOTES.txt

@@ -1,5 +1,5 @@
 Chart version: {{ .Chart.Version }}
-Kyverno version: {{ default .Chart.AppVersion (default .Values.image.tag .Values.admissionController.initContainer.image.tag) }}
+Kyverno version: {{ default .Chart.AppVersion (default .Values.admissionController.container.image.tag .Values.admissionController.initContainer.image.tag) }}
 
 Thank you for installing {{ .Chart.Name }}! Your release is named {{ .Release.Name }}.
 

charts/kyverno/templates/admission-controller/deployment.yaml

@@ -26,7 +26,7 @@ spec:
       annotations: {{ tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
     spec:
-      {{- with .Values.image.pullSecrets }}
+      {{- with .Values.admissionController.container.image.pullSecrets }}
       imagePullSecrets:
         {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
@@ -78,8 +78,8 @@ spec:
         {{- toYaml .Values.extraInitContainers | nindent 8 }}
       {{- end }}
         - name: kyverno-pre
-          image: {{ include "kyverno.image" (dict "image" .Values.admissionController.initContainer.image "defaultTag" (default .Chart.AppVersion .Values.image.tag)) | quote }}
-          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.admissionController.initContainer.image.pullPolicy }}
+          image: {{ include "kyverno.image" (dict "image" .Values.admissionController.initContainer.image "defaultTag" (default .Chart.AppVersion .Values.admissionController.container.image.tag)) | quote }}
+          imagePullPolicy: {{ default .Values.admissionController.container.image.pullPolicy .Values.admissionController.initContainer.image.pullPolicy }}
           {{- if .Values.initContainer.extraArgs }}
           args:
               {{- tpl (toYaml .Values.initContainer.extraArgs) . | nindent 12 }}
@@ -113,11 +113,11 @@ spec:
         {{- toYaml .Values.extraContainers | nindent 8 }}
       {{- end }}
         - name: kyverno
-          image: {{ include "kyverno.image" (dict "image" .Values.image "defaultTag" .Chart.AppVersion) | quote }}
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          image: {{ include "kyverno.image" (dict "image" .Values.admissionController.container.image "defaultTag" .Chart.AppVersion) | quote }}
+          imagePullPolicy: {{ .Values.admissionController.container.image.pullPolicy }}
           {{- if or .Values.extraArgs .Values.imagePullSecrets }}
           args:
-            - --servicePort={{ .Values.service.port }}
+            - --servicePort={{ .Values.admissionController.service.port }}
             {{- if .Values.extraArgs -}}
               {{ tpl (toYaml .Values.extraArgs) . | nindent 12 }}
             {{- end }}
@@ -125,7 +125,7 @@ spec:
             - --imagePullSecrets={{- join "," (concat (keys .Values.imagePullSecrets) .Values.existingImagePullSecrets) }}
             {{- end }}
           {{- end }}
-          {{- with .Values.resources }}
+          {{- with .Values.admissionController.container.resources }}
           resources: {{ tpl (toYaml .) $ | nindent 12 }}
           {{- end }}
           {{- with .Values.securityContext }}

charts/kyverno/templates/admission-controller/service.yaml

@@ -5,23 +5,23 @@ metadata:
   namespace: {{ template "kyverno.namespace" . }}
   labels:
     {{- include "kyverno.admission-controller.labels" . | nindent 4 }}
-  {{- with .Values.service.annotations }}
+  {{- with .Values.admissionController.service.annotations }}
   annotations: {{ tpl (toYaml .) $ | nindent 4 }}
   {{- end }}
 spec:
   ports:
-  - port: {{ .Values.service.port }}
+  - port: {{ .Values.admissionController.service.port }}
     targetPort: https
     protocol: TCP
     name: https
-    {{- if and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort)) }}
-    nodePort: {{ .Values.service.nodePort }}
+    {{- if and (eq .Values.admissionController.service.type "NodePort") (not (empty .Values.admissionController.service.nodePort)) }}
+    nodePort: {{ .Values.admissionController.service.nodePort }}
     {{- end }}
   selector:
     {{- include "kyverno.admission-controller.matchLabels" . | nindent 4 }}
-  type: {{ .Values.service.type }}
+  type: {{ .Values.admissionController.service.type }}
 ---
-{{- if .Values.metricsService.create }}
+{{- if .Values.admissionController.metricsService.create }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -29,19 +29,19 @@ metadata:
   namespace: {{ template "kyverno.namespace" . }}
   labels:
     {{- include "kyverno.admission-controller.labels" . | nindent 4 }}
-  {{- with .Values.metricsService.annotations }}
+  {{- with .Values.admissionController.metricsService.annotations }}
   annotations: {{ tpl (toYaml .) $ | nindent 4 }}
   {{- end }}
 spec:
   ports:
-  - port: {{ .Values.metricsService.port }}
+  - port: {{ .Values.admissionController.metricsService.port }}
     targetPort: 8000
     protocol: TCP
     name: metrics-port
-    {{- if and (eq .Values.metricsService.type "NodePort") (not (empty .Values.metricsService.nodePort)) }}
-    nodePort: {{ .Values.metricsService.nodePort }}
+    {{- if and (eq .Values.admissionController.metricsService.type "NodePort") (not (empty .Values.admissionController.metricsService.nodePort)) }}
+    nodePort: {{ .Values.admissionController.metricsService.nodePort }}
     {{- end }}
   selector:
     {{- include "kyverno.admission-controller.matchLabels" . | nindent 4 }}
-  type: {{ .Values.metricsService.type }}
+  type: {{ .Values.admissionController.metricsService.type }}
 {{- end -}}

charts/kyverno/templates/networkpolicy.yaml

@@ -22,9 +22,9 @@ spec:
     - protocol: TCP
       port: 9443 # webhook access
   # Allow prometheus scrapes for metrics
-  {{- if .Values.metricsService.create }}
+  {{- if .Values.admissionController.metricsService.create }}
     - protocol: TCP
-      port: {{ .Values.metricsService.port }}
+      port: {{ .Values.admissionController.metricsService.port }}
   {{- end }}
   {{- else }}
   ingress:

charts/kyverno/templates/tests/_helpers.tpl

@@ -23,5 +23,5 @@ helm.sh/hook: test
 {{- end -}}
 
 {{- define "kyverno.test.imagePullPolicy" -}}
-{{- default .Values.image.pullPolicy .Values.test.image.pullPolicy -}}
+{{- default .Values.admissionController.container.image.pullPolicy .Values.test.image.pullPolicy -}}
 {{- end -}}

charts/kyverno/templates/tests/admission-controller-liveness.yaml

@@ -24,4 +24,4 @@ spec:
       command:
         - /bin/sh
         - -c
-        - sleep 20 ; wget -O- -S --no-check-certificate https://{{ template "kyverno.admission-controller.serviceName" . }}.{{ template "kyverno.namespace" . }}:{{ .Values.service.port }}/health/liveness
+        - sleep 20 ; wget -O- -S --no-check-certificate https://{{ template "kyverno.admission-controller.serviceName" . }}.{{ template "kyverno.namespace" . }}:{{ .Values.admissionController.service.port }}/health/liveness

charts/kyverno/templates/tests/admission-controller-metrics.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.metricsService.create -}}
+{{- if .Values.admissionController.metricsService.create -}}
 apiVersion: v1
 kind: Pod
 metadata:
@@ -25,5 +25,5 @@ spec:
       command:
         - /bin/sh
         - -c
-        - sleep 20 ; wget -O- -S --no-check-certificate http://{{ template "kyverno.admission-controller.serviceName" . }}-metrics.{{ template "kyverno.namespace" . }}:{{ .Values.metricsService.port }}/metrics
+        - sleep 20 ; wget -O- -S --no-check-certificate http://{{ template "kyverno.admission-controller.serviceName" . }}-metrics.{{ template "kyverno.namespace" . }}:{{ .Values.admissionController.metricsService.port }}/metrics
 {{- end -}}

charts/kyverno/templates/tests/admission-controller-readiness.yaml

@@ -24,4 +24,4 @@ spec:
       command:
         - /bin/sh
         - -c
-        - sleep 20 ; wget -O- -S --no-check-certificate https://{{ template "kyverno.admission-controller.serviceName" . }}.{{ template "kyverno.namespace" . }}:{{ .Values.service.port }}/health/readiness
+        - sleep 20 ; wget -O- -S --no-check-certificate https://{{ template "kyverno.admission-controller.serviceName" . }}.{{ template "kyverno.namespace" . }}:{{ .Values.admissionController.service.port }}/health/readiness

charts/kyverno/values.yaml

@@ -203,20 +203,6 @@ rbac:
     annotations: {}
       # example.com/annotation: value
 
-image:
-  # -- Image registry
-  registry: ghcr.io
-  # -- Image repository
-  repository: kyverno/kyverno
-  # -- (string) Image tag
-  # Defaults to appVersion in Chart.yaml if omitted
-  tag: ~
-  # -- Image pull policy
-  pullPolicy: IfNotPresent
-  # -- Image pull secrets
-  pullSecrets: []
-  # - secretName
-
 initContainer:
   # -- Extra arguments to give to the kyvernopre binary.
   extraArgs:
@@ -266,15 +252,6 @@ extraContainers: []
 #   image: busybox
 #   command: ['sh', '-c', 'echo Hello && sleep 3600']
 
-resources:
-  # -- Pod resource limits
-  limits:
-    memory: 384Mi
-  # -- Pod resource requests
-  requests:
-    cpu: 100m
-    memory: 128Mi
-
 # -- Additional resources to be added to controller RBAC permissions.
 generatecontrollerExtraResources: []
 # - ResourceA
@@ -288,31 +265,6 @@ excludeKyvernoNamespace: true
 # Namespaces to exclude from the default resourceFilters
 resourceFiltersExcludeNamespaces: []
 
-service:
-  # -- Service port.
-  port: 443
-  # -- Service type.
-  type: ClusterIP
-  # -- Service node port.
-  # Only used if `service.type` is `NodePort`.
-  nodePort:
-  # -- Service annotations.
-  annotations: {}
-
-metricsService:
-  # -- Create service.
-  create: true
-  # -- Service port.
-  # Kyverno's metrics server will be exposed at this port.
-  port: 8000
-  # -- Service type.
-  type: ClusterIP
-  # -- Service node port.
-  # Only used if `metricsService.type` is `NodePort`.
-  nodePort:
-  # -- Service annotations.
-  annotations: {}
-
 networkPolicy:
   # -- When true, use a NetworkPolicy to allow ingress to the webhook
   # This is useful on clusters using Calico and/or native k8s network policies in a default-deny setup.
@@ -483,6 +435,10 @@ admissionController:
   sigstoreVolume:
     emptyDir: {}
 
+  # -- Image pull secrets
+  pullSecrets: []
+    # - secretName
+
   initContainer:
 
     image:
@@ -507,6 +463,53 @@ admissionController:
         cpu: 10m
         memory: 64Mi
 
+  container:
+
+    image:
+      # -- Image registry
+      registry: ghcr.io
+      # -- Image repository
+      repository: kyverno/kyverno
+      # -- (string) Image tag
+      # Defaults to appVersion in Chart.yaml if omitted
+      tag: ~
+      # -- Image pull policy
+      pullPolicy: IfNotPresent
+
+    resources:
+      # -- Pod resource limits
+      limits:
+        memory: 384Mi
+      # -- Pod resource requests
+      requests:
+        cpu: 100m
+        memory: 128Mi
+
+  service:
+    # -- Service port.
+    port: 443
+    # -- Service type.
+    type: ClusterIP
+    # -- Service node port.
+    # Only used if `type` is `NodePort`.
+    nodePort:
+    # -- Service annotations.
+    annotations: {}
+
+  metricsService:
+    # -- Create service.
+    create: true
+    # -- Service port.
+    # Kyverno's metrics server will be exposed at this port.
+    port: 8000
+    # -- Service type.
+    type: ClusterIP
+    # -- Service node port.
+    # Only used if `type` is `NodePort`.
+    nodePort:
+    # -- Service annotations.
+    annotations: {}
+
 # Cleanup controller configuration
 cleanupController: