From 5c9273de84a6bbec6d97b2d863273a5a00635c8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Thu, 2 Mar 2023 17:23:22 +0100 Subject: [PATCH] refactor: helm admission controller config (#6457) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: helm admission controller config Signed-off-by: Charles-Edouard Brétéché * pdb Signed-off-by: Charles-Edouard Brétéché * refactor: helm admission controller config Signed-off-by: Charles-Edouard Brétéché * certs Signed-off-by: Charles-Edouard Brétéché --------- Signed-off-by: Charles-Edouard Brétéché --- charts/kyverno/README.md | 29 ++-- charts/kyverno/README.md.gotmpl | 13 +- .../admission-controller/deployment.yaml | 31 ++-- .../admission-controller/secret.yaml | 2 +- charts/kyverno/values.yaml | 152 +++++++++--------- 5 files changed, 120 insertions(+), 107 deletions(-) diff --git a/charts/kyverno/README.md b/charts/kyverno/README.md index 5894ee0a27..a400c18360 100644 --- a/charts/kyverno/README.md +++ b/charts/kyverno/README.md @@ -130,6 +130,15 @@ In `v3` chart values changed significantly, please read the instructions below t - `tolerations` has been replaced with `admissionController.tolerations` - `topologySpreadConstraints` has been replaced with `admissionController.topologySpreadConstraints` - `podDisruptionBudget` has been replaced with `admissionController.podDisruptionBudget` +- `antiAffinity` has been replaced with `admissionController.antiAffinity` +- `antiAffinity.enable` has been replaced with `admissionController.antiAffinity.enabled` +- `podAntiAffinity` has been replaced with `admissionController.podAntiAffinity` +- `podAffinity` has been replaced with `admissionController.podAffinity` +- `nodeAffinity` has been replaced with `admissionController.nodeAffinity` +- `startupProbe` has been replaced with `admissionController.startupProbe` +- `livenessProbe` has been replaced with `admissionController.livenessProbe` +- `readinessProbe` has been replaced with `admissionController.readinessProbe` +- `createSelfSignedCert` has been replaced with `admissionController.createSelfSignedCert` - Labels and selectors have been reworked and due to immutability, upgrading from `v2` to `v3` is going to be rejected. The easiest solution is to uninstall `v2` and reinstall `v3` once values have been adapted to the changes described above. @@ -200,10 +209,6 @@ The command removes all the Kubernetes components associated with the chart and | podAnnotations | object | `{}` | Additional annotations to add to each pod | | podSecurityContext | object | `{}` | Security context for the pod | | securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}` | Security context for the containers | -| antiAffinity.enable | bool | `true` | Pod antiAffinities toggle. Enabled by default but can be disabled if you want to schedule pods to the same node. | -| podAntiAffinity | object | See [values.yaml](values.yaml) | Pod anti affinity constraints. | -| podAffinity | object | `{}` | Pod affinity constraints. | -| nodeAffinity | object | `{}` | Node affinity constraints. | | envVarsInit | object | `{}` | Env variables for initContainers. | | envVars | object | `{}` | Env variables for containers. | | extraArgs | list | `["--loggingFormat=text"]` | Extra arguments to give to the binary. | @@ -213,9 +218,6 @@ The command removes all the Kubernetes components associated with the chart and | resources.requests | object | `{"cpu":"100m","memory":"128Mi"}` | Pod resource requests | | initResources.limits | object | `{"cpu":"100m","memory":"256Mi"}` | Pod resource limits | | initResources.requests | object | `{"cpu":"10m","memory":"64Mi"}` | Pod resource requests | -| startupProbe | object | See [values.yaml](values.yaml) | Startup probe. The block is directly forwarded into the deployment, so you can use whatever startupProbes configuration you want. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ | -| livenessProbe | object | See [values.yaml](values.yaml) | Liveness probe. The block is directly forwarded into the deployment, so you can use whatever livenessProbe configuration you want. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ | -| readinessProbe | object | See [values.yaml](values.yaml) | Readiness Probe. The block is directly forwarded into the deployment, so you can use whatever readinessProbe configuration you want. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ | | generatecontrollerExtraResources | list | `[]` | Additional resources to be added to controller RBAC permissions. | | excludeKyvernoNamespace | bool | `true` | Exclude Kyverno namespace Determines if default Kyverno namespace exclusion is enabled for webhooks and resourceFilters | | resourceFiltersExcludeNamespaces | list | `[]` | resourceFilter namespace exclude Namespaces to exclude from the default resourceFilters | @@ -235,7 +237,6 @@ The command removes all the Kubernetes components associated with the chart and | serviceMonitor.scrapeTimeout | string | `"25s"` | Timeout if metrics can't be retrieved in given time interval | | serviceMonitor.secure | bool | `false` | Is TLS required for endpoint | | serviceMonitor.tlsConfig | object | `{}` | TLS Configuration for endpoint | -| createSelfSignedCert | bool | `false` | Kyverno requires a certificate key pair and corresponding certificate authority to properly register its webhooks. This can be done in one of 3 ways: 1) Use kube-controller-manager to generate a CA-signed certificate (preferred) 2) Provide your own CA and cert. In this case, you will need to create a certificate with a specific name and data structure. As long as you follow the naming scheme, it will be automatically picked up. kyverno-svc.(namespace).svc.kyverno-tls-ca (with data entries named tls.key and tls.crt) kyverno-svc.kyverno.svc.kyverno-tls-pair (with data entries named tls.key and tls.crt) 3) Let Helm generate a self signed cert, by setting createSelfSignedCert true If letting Kyverno create its own CA or providing your own, make createSelfSignedCert is false | | networkPolicy.enabled | bool | `false` | When true, use a NetworkPolicy to allow ingress to the webhook This is useful on clusters using Calico and/or native k8s network policies in a default-deny setup. | | networkPolicy.ingressFrom | list | `[]` | A list of valid from selectors according to https://kubernetes.io/docs/concepts/services-networking/network-policies. | | webhooksCleanup.enabled | bool | `false` | Create a helm pre-delete hook to cleanup webhooks. | @@ -246,13 +247,21 @@ The command removes all the Kubernetes components associated with the chart and | grafana.configMapName | string | `"{{ include \"kyverno.fullname\" . }}-grafana"` | Configmap name template. | | grafana.namespace | string | `nil` | Namespace to create the grafana dashboard configmap. If not set, it will be created in the same namespace where the chart is deployed. | | grafana.annotations | object | `{}` | Grafana dashboard configmap annotations. | +| admissionController.createSelfSignedCert | bool | `false` | Create self-signed certificates at deployment time. The certificates won't be automatically renewed if this is set to `true`. | | admissionController.replicas | int | `nil` | Desired number of pods | | admissionController.updateStrategy | object | See [values.yaml](values.yaml) | Deployment update strategy. Ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy | | admissionController.priorityClassName | string | `""` | Optional priority class | | admissionController.hostNetwork | bool | `false` | Change `hostNetwork` to `true` when you want the pod to share its host's network namespace. Useful for situations like when you end up dealing with a custom CNI over Amazon EKS. Update the `dnsPolicy` accordingly as well to suit the host network mode. | | admissionController.dnsPolicy | string | `"ClusterFirst"` | `dnsPolicy` determines the manner in which DNS resolution happens in the cluster. In case of `hostNetwork: true`, usually, the `dnsPolicy` is suitable to be `ClusterFirstWithHostNet`. For further reference: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy. | +| admissionController.startupProbe | object | See [values.yaml](values.yaml) | Startup probe. The block is directly forwarded into the deployment, so you can use whatever startupProbes configuration you want. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ | +| admissionController.livenessProbe | object | See [values.yaml](values.yaml) | Liveness probe. The block is directly forwarded into the deployment, so you can use whatever livenessProbe configuration you want. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ | +| admissionController.readinessProbe | object | See [values.yaml](values.yaml) | Readiness Probe. The block is directly forwarded into the deployment, so you can use whatever readinessProbe configuration you want. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ | | admissionController.nodeSelector | object | `{}` | Node labels for pod assignment | | admissionController.tolerations | list | `[]` | List of node taints to tolerate | +| admissionController.antiAffinity.enabled | bool | `true` | Pod antiAffinities toggle. Enabled by default but can be disabled if you want to schedule pods to the same node. | +| admissionController.podAntiAffinity | object | See [values.yaml](values.yaml) | Pod anti affinity constraints. | +| admissionController.podAffinity | object | `{}` | Pod affinity constraints. | +| admissionController.nodeAffinity | object | `{}` | Node affinity constraints. | | admissionController.topologySpreadConstraints | list | `[]` | Topology spread constraints. | | admissionController.podDisruptionBudget.minAvailable | int | `1` | Configures the minimum available pods for disruptions. Cannot be used if `maxUnavailable` is set. | | admissionController.podDisruptionBudget.maxUnavailable | string | `nil` | Configures the maximum unavailable pods for disruptions. Cannot be used if `minAvailable` is set. | @@ -417,9 +426,9 @@ The command removes all the Kubernetes components associated with the chart and ## TLS Configuration -If `createSelfSignedCert` is `true`, Helm will take care of the steps of creating an external self-signed certificate described in option 2 of the [installation documentation](https://kyverno.io/docs/installation/#option-2-use-your-own-ca-signed-certificate) +If `admissionController.createSelfSignedCert` is `true`, Helm will take care of the steps of creating an external self-signed certificate described in option 2 of the [installation documentation](https://kyverno.io/docs/installation/#option-2-use-your-own-ca-signed-certificate) -If `createSelfSignedCert` is `false`, Kyverno will generate a self-signed CA and a certificate, or you can provide your own TLS CA and signed-key pair and create the secret yourself as described in the [documentation](https://kyverno.io/docs/installation/#customize-the-installation-of-kyverno). +If `admissionController.createSelfSignedCert` is `false`, Kyverno will generate a self-signed CA and a certificate, or you can provide your own TLS CA and signed-key pair and create the secret yourself as described in the [documentation](https://kyverno.io/docs/installation/#customize-the-installation-of-kyverno). ## Default resource filters diff --git a/charts/kyverno/README.md.gotmpl b/charts/kyverno/README.md.gotmpl index f7040b9016..e10d64290c 100644 --- a/charts/kyverno/README.md.gotmpl +++ b/charts/kyverno/README.md.gotmpl @@ -130,6 +130,15 @@ In `v3` chart values changed significantly, please read the instructions below t - `tolerations` has been replaced with `admissionController.tolerations` - `topologySpreadConstraints` has been replaced with `admissionController.topologySpreadConstraints` - `podDisruptionBudget` has been replaced with `admissionController.podDisruptionBudget` +- `antiAffinity` has been replaced with `admissionController.antiAffinity` +- `antiAffinity.enable` has been replaced with `admissionController.antiAffinity.enabled` +- `podAntiAffinity` has been replaced with `admissionController.podAntiAffinity` +- `podAffinity` has been replaced with `admissionController.podAffinity` +- `nodeAffinity` has been replaced with `admissionController.nodeAffinity` +- `startupProbe` has been replaced with `admissionController.startupProbe` +- `livenessProbe` has been replaced with `admissionController.livenessProbe` +- `readinessProbe` has been replaced with `admissionController.readinessProbe` +- `createSelfSignedCert` has been replaced with `admissionController.createSelfSignedCert` - Labels and selectors have been reworked and due to immutability, upgrading from `v2` to `v3` is going to be rejected. The easiest solution is to uninstall `v2` and reinstall `v3` once values have been adapted to the changes described above. @@ -151,9 +160,9 @@ The command removes all the Kubernetes components associated with the chart and ## TLS Configuration -If `createSelfSignedCert` is `true`, Helm will take care of the steps of creating an external self-signed certificate described in option 2 of the [installation documentation](https://kyverno.io/docs/installation/#option-2-use-your-own-ca-signed-certificate) +If `admissionController.createSelfSignedCert` is `true`, Helm will take care of the steps of creating an external self-signed certificate described in option 2 of the [installation documentation](https://kyverno.io/docs/installation/#option-2-use-your-own-ca-signed-certificate) -If `createSelfSignedCert` is `false`, Kyverno will generate a self-signed CA and a certificate, or you can provide your own TLS CA and signed-key pair and create the secret yourself as described in the [documentation](https://kyverno.io/docs/installation/#customize-the-installation-of-kyverno). +If `admissionController.createSelfSignedCert` is `false`, Kyverno will generate a self-signed CA and a certificate, or you can provide your own TLS CA and signed-key pair and create the secret yourself as described in the [documentation](https://kyverno.io/docs/installation/#customize-the-installation-of-kyverno). ## Default resource filters diff --git a/charts/kyverno/templates/admission-controller/deployment.yaml b/charts/kyverno/templates/admission-controller/deployment.yaml index 9813ceb637..94143d306b 100644 --- a/charts/kyverno/templates/admission-controller/deployment.yaml +++ b/charts/kyverno/templates/admission-controller/deployment.yaml @@ -55,19 +55,19 @@ spec: {{- with .Values.admissionController.dnsPolicy }} dnsPolicy: {{ . }} {{- end }} - {{- if or .Values.antiAffinity.enable .Values.podAffinity .Values.nodeAffinity }} + {{- if or .Values.admissionController.antiAffinity.enable .Values.admissionController.podAffinity .Values.admissionController.nodeAffinity }} affinity: - {{- if .Values.antiAffinity.enable }} - {{- with .Values.podAntiAffinity }} + {{- if .Values.admissionController.antiAffinity.enabled }} + {{- with .Values.admissionController.podAntiAffinity }} podAntiAffinity: {{- tpl (toYaml .) $ | nindent 10 }} {{- end }} {{- end }} - {{- with .Values.podAffinity }} + {{- with .Values.admissionController.podAffinity }} podAffinity: {{- tpl (toYaml .) $ | nindent 10 }} {{- end }} - {{- with .Values.nodeAffinity }} + {{- with .Values.admissionController.nodeAffinity }} nodeAffinity: {{- tpl (toYaml .) $ | nindent 10 }} {{- end }} @@ -162,15 +162,18 @@ spec: {{- end }} - name: KYVERNO_DEPLOYMENT value: {{ template "kyverno.fullname" . }} - {{- with .Values.startupProbe }} - startupProbe: {{ tpl (toYaml .) $ | nindent 12 }} - {{- end }} - {{- with .Values.livenessProbe }} - livenessProbe: {{ tpl (toYaml .) $ | nindent 12 }} - {{- end }} - {{- with .Values.readinessProbe }} - readinessProbe: {{ tpl (toYaml .) $ | nindent 12 }} - {{- end }} + {{- with .Values.admissionController.startupProbe }} + startupProbe: + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- with .Values.admissionController.livenessProbe }} + livenessProbe: + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- with .Values.admissionController.readinessProbe }} + readinessProbe: + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} volumeMounts: - mountPath: {{ .Values.tufRootMountPath }} name: sigstore diff --git a/charts/kyverno/templates/admission-controller/secret.yaml b/charts/kyverno/templates/admission-controller/secret.yaml index e16734156e..1c6b7182a4 100644 --- a/charts/kyverno/templates/admission-controller/secret.yaml +++ b/charts/kyverno/templates/admission-controller/secret.yaml @@ -1,4 +1,4 @@ -{{- if .Values.createSelfSignedCert -}} +{{- if .Values.admissionController.createSelfSignedCert -}} {{- $ca := genCA (printf "*.%s.svc" (include "kyverno.namespace" .)) 1024 -}} {{- $svcName := (printf "%s.%s.svc" (include "kyverno.admission-controller.serviceName" .) (include "kyverno.namespace" .)) -}} {{- $cert := genSignedCert $svcName nil (list $svcName) 1024 $ca -}} diff --git a/charts/kyverno/values.yaml b/charts/kyverno/values.yaml index 0c4642a14b..0012f3f53b 100644 --- a/charts/kyverno/values.yaml +++ b/charts/kyverno/values.yaml @@ -257,31 +257,6 @@ securityContext: seccompProfile: type: RuntimeDefault -antiAffinity: - # -- Pod antiAffinities toggle. - # Enabled by default but can be disabled if you want to schedule pods to the same node. - enable: true - -# -- Pod anti affinity constraints. -# @default -- See [values.yaml](values.yaml) -podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app.kubernetes.io/name - operator: In - values: - - '{{ template "kyverno.name" . }}' - topologyKey: kubernetes.io/hostname - -# -- Pod affinity constraints. -podAffinity: {} - -# -- Node affinity constraints. -nodeAffinity: {} - # -- Env variables for initContainers. envVarsInit: {} @@ -325,49 +300,6 @@ initResources: cpu: 10m memory: 64Mi -# -- Startup probe. -# The block is directly forwarded into the deployment, so you can use whatever startupProbes configuration you want. -# ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ -# @default -- See [values.yaml](values.yaml) -startupProbe: - httpGet: - path: /health/liveness - port: 9443 - scheme: HTTPS - failureThreshold: 20 - initialDelaySeconds: 2 - periodSeconds: 6 - -# -- Liveness probe. -# The block is directly forwarded into the deployment, so you can use whatever livenessProbe configuration you want. -# ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ -# @default -- See [values.yaml](values.yaml) -livenessProbe: - httpGet: - path: /health/liveness - port: 9443 - scheme: HTTPS - initialDelaySeconds: 15 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 2 - successThreshold: 1 - -# -- Readiness Probe. -# The block is directly forwarded into the deployment, so you can use whatever readinessProbe configuration you want. -# ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ -# @default -- See [values.yaml](values.yaml) -readinessProbe: - httpGet: - path: /health/readiness - port: 9443 - scheme: HTTPS - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 6 - successThreshold: 1 - # -- Additional resources to be added to controller RBAC permissions. generatecontrollerExtraResources: [] # - ResourceA @@ -423,18 +355,6 @@ serviceMonitor: # -- TLS Configuration for endpoint tlsConfig: {} -# -- Kyverno requires a certificate key pair and corresponding certificate authority -# to properly register its webhooks. This can be done in one of 3 ways: -# 1) Use kube-controller-manager to generate a CA-signed certificate (preferred) -# 2) Provide your own CA and cert. -# In this case, you will need to create a certificate with a specific name and data structure. -# As long as you follow the naming scheme, it will be automatically picked up. -# kyverno-svc.(namespace).svc.kyverno-tls-ca (with data entries named tls.key and tls.crt) -# kyverno-svc.kyverno.svc.kyverno-tls-pair (with data entries named tls.key and tls.crt) -# 3) Let Helm generate a self signed cert, by setting createSelfSignedCert true -# If letting Kyverno create its own CA or providing your own, make createSelfSignedCert is false -createSelfSignedCert: false - networkPolicy: # -- When true, use a NetworkPolicy to allow ingress to the webhook # This is useful on clusters using Calico and/or native k8s network policies in a default-deny setup. @@ -472,6 +392,10 @@ grafana: # Admission controller configuration admissionController: + # -- Create self-signed certificates at deployment time. + # The certificates won't be automatically renewed if this is set to `true`. + createSelfSignedCert: false + # -- (int) Desired number of pods replicas: ~ @@ -497,12 +421,80 @@ admissionController: # For further reference: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy. dnsPolicy: ClusterFirst + # -- Startup probe. + # The block is directly forwarded into the deployment, so you can use whatever startupProbes configuration you want. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ + # @default -- See [values.yaml](values.yaml) + startupProbe: + httpGet: + path: /health/liveness + port: 9443 + scheme: HTTPS + failureThreshold: 20 + initialDelaySeconds: 2 + periodSeconds: 6 + + # -- Liveness probe. + # The block is directly forwarded into the deployment, so you can use whatever livenessProbe configuration you want. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ + # @default -- See [values.yaml](values.yaml) + livenessProbe: + httpGet: + path: /health/liveness + port: 9443 + scheme: HTTPS + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 2 + successThreshold: 1 + + # -- Readiness Probe. + # The block is directly forwarded into the deployment, so you can use whatever readinessProbe configuration you want. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ + # @default -- See [values.yaml](values.yaml) + readinessProbe: + httpGet: + path: /health/readiness + port: 9443 + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + # -- Node labels for pod assignment nodeSelector: {} # -- List of node taints to tolerate tolerations: [] + antiAffinity: + # -- Pod antiAffinities toggle. + # Enabled by default but can be disabled if you want to schedule pods to the same node. + enabled: true + + # -- Pod anti affinity constraints. + # @default -- See [values.yaml](values.yaml) + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: + - admission-controller + topologyKey: kubernetes.io/hostname + + # -- Pod affinity constraints. + podAffinity: {} + + # -- Node affinity constraints. + nodeAffinity: {} + # -- Topology spread constraints. topologySpreadConstraints: []