From 3adadae7da859d9bba60b1c64da9e38f2476a102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Mon, 20 Mar 2023 06:42:34 +0100 Subject: [PATCH] refactoring: helm logging, tracing and metering config (#6613) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Charles-Edouard Brétéché Co-authored-by: shuting --- charts/kyverno/README.md | 29 +++++--- charts/kyverno/README.md.gotmpl | 1 + .../admission-controller/deployment.yaml | 21 ++++++ charts/kyverno/values.yaml | 66 ++++++++++++------- scripts/config/dev/kyverno.yaml | 20 +++--- scripts/config/standard/kyverno.yaml | 6 -- 6 files changed, 96 insertions(+), 47 deletions(-) diff --git a/charts/kyverno/README.md b/charts/kyverno/README.md index d18b58a92f..208e37c4cd 100644 --- a/charts/kyverno/README.md +++ b/charts/kyverno/README.md @@ -165,6 +165,7 @@ In `v3` chart values changed significantly, please read the instructions below t - `generatecontrollerExtraResources` has been replaced with `admissionController.rbac.clusterRole.extraResources` - `networkPolicy` has been replaced with `admissionController.networkPolicy` - all `extraArgs` now use objects instead of arrays +- logging, tracing and metering are now configured using `*Controller.logging`, `*Controller.tracing` and `*Controller.metering` - Labels and selectors have been reworked and due to immutability, upgrading from `v2` to `v3` is going to be rejected. The easiest solution is to uninstall `v2` and reinstall `v3` once values have been adapted to the changes described above. @@ -252,13 +253,6 @@ The command removes all the Kubernetes components associated with the chart and | admissionController.podSecurityContext | object | `{}` | Security context for the pod | | admissionController.podDisruptionBudget.minAvailable | int | `1` | Configures the minimum available pods for disruptions. Cannot be used if `maxUnavailable` is set. | | admissionController.podDisruptionBudget.maxUnavailable | string | `nil` | Configures the maximum unavailable pods for disruptions. Cannot be used if `minAvailable` is set. | -| admissionController.serviceMonitor.enabled | bool | `false` | Create a `ServiceMonitor` to collect Prometheus metrics. | -| admissionController.serviceMonitor.additionalLabels | object | `{}` | Additional labels | -| admissionController.serviceMonitor.namespace | string | `nil` | Override namespace | -| admissionController.serviceMonitor.interval | string | `"30s"` | Interval to scrape metrics | -| admissionController.serviceMonitor.scrapeTimeout | string | `"25s"` | Timeout if metrics can't be retrieved in given time interval | -| admissionController.serviceMonitor.secure | bool | `false` | Is TLS required for endpoint | -| admissionController.serviceMonitor.tlsConfig | object | `{}` | TLS Configuration for endpoint | | admissionController.tufRootMountPath | string | `"/.sigstore"` | A writable volume to use for the TUF root initialization. | | admissionController.sigstoreVolume | object | `{"emptyDir":{}}` | Volume to be mounted in pods for TUF/cosign work. | | admissionController.pullSecrets | list | `[]` | Image pull secrets | @@ -269,7 +263,7 @@ The command removes all the Kubernetes components associated with the chart and | admissionController.initContainer.resources.limits | object | `{"cpu":"100m","memory":"256Mi"}` | Pod resource limits | | admissionController.initContainer.resources.requests | object | `{"cpu":"10m","memory":"64Mi"}` | Pod resource requests | | admissionController.initContainer.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}` | Container security context | -| admissionController.initContainer.extraArgs | object | `{"loggingFormat":"text"}` | Additional container args. | +| admissionController.initContainer.extraArgs | object | `{}` | Additional container args. | | admissionController.initContainer.extraEnvVars | list | `[]` | Additional container environment variables. | | admissionController.container.image.registry | string | `"ghcr.io"` | Image registry | | admissionController.container.image.repository | string | `"kyverno/kyverno"` | Image repository | @@ -278,7 +272,7 @@ The command removes all the Kubernetes components associated with the chart and | admissionController.container.resources.limits | object | `{"memory":"384Mi"}` | Pod resource limits | | admissionController.container.resources.requests | object | `{"cpu":"100m","memory":"128Mi"}` | Pod resource requests | | admissionController.container.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}` | Container security context | -| admissionController.container.extraArgs | object | `{"loggingFormat":"text"}` | Additional container args. | +| admissionController.container.extraArgs | object | `{}` | Additional container args. | | admissionController.container.extraEnvVars | list | `[]` | Additional container environment variables. | | admissionController.extraInitContainers | list | `[]` | Array of extra init containers | | admissionController.extraContainers | list | `[]` | Array of extra containers to run alongside kyverno | @@ -293,6 +287,23 @@ The command removes all the Kubernetes components associated with the chart and | admissionController.metricsService.annotations | object | `{}` | Service annotations. | | admissionController.networkPolicy.enabled | bool | `false` | When true, use a NetworkPolicy to allow ingress to the webhook This is useful on clusters using Calico and/or native k8s network policies in a default-deny setup. | | admissionController.networkPolicy.ingressFrom | list | `[]` | A list of valid from selectors according to https://kubernetes.io/docs/concepts/services-networking/network-policies. | +| admissionController.serviceMonitor.enabled | bool | `false` | Create a `ServiceMonitor` to collect Prometheus metrics. | +| admissionController.serviceMonitor.additionalLabels | object | `{}` | Additional labels | +| admissionController.serviceMonitor.namespace | string | `nil` | Override namespace | +| admissionController.serviceMonitor.interval | string | `"30s"` | Interval to scrape metrics | +| admissionController.serviceMonitor.scrapeTimeout | string | `"25s"` | Timeout if metrics can't be retrieved in given time interval | +| admissionController.serviceMonitor.secure | bool | `false` | Is TLS required for endpoint | +| admissionController.serviceMonitor.tlsConfig | object | `{}` | TLS Configuration for endpoint | +| admissionController.tracing.enabled | bool | `false` | Enable tracing | +| admissionController.tracing.address | string | `nil` | Traces receiver address | +| admissionController.tracing.port | string | `nil` | Traces receiver port | +| admissionController.tracing.creds | string | `""` | Traces receiver credentials | +| admissionController.logging.format | string | `"text"` | Logging format | +| admissionController.metering.disabled | bool | `false` | Disable metrics export | +| admissionController.metering.config | string | `"prometheus"` | Otel configuration, can be `prometheus` or `grpc` | +| admissionController.metering.port | int | `8000` | Prometheus endpoint port | +| admissionController.metering.collector | string | `""` | Otel collector endpoint | +| admissionController.metering.creds | string | `""` | Otel collector credentials | | cleanupController.enabled | bool | `true` | Enable cleanup controller. | | cleanupController.rbac.create | bool | `true` | Create RBAC resources | | cleanupController.rbac.serviceAccount.name | string | `nil` | Service account name | diff --git a/charts/kyverno/README.md.gotmpl b/charts/kyverno/README.md.gotmpl index f54c2756f4..b9bd9ed21c 100644 --- a/charts/kyverno/README.md.gotmpl +++ b/charts/kyverno/README.md.gotmpl @@ -165,6 +165,7 @@ In `v3` chart values changed significantly, please read the instructions below t - `generatecontrollerExtraResources` has been replaced with `admissionController.rbac.clusterRole.extraResources` - `networkPolicy` has been replaced with `admissionController.networkPolicy` - all `extraArgs` now use objects instead of arrays +- logging, tracing and metering are now configured using `*Controller.logging`, `*Controller.tracing` and `*Controller.metering` - Labels and selectors have been reworked and due to immutability, upgrading from `v2` to `v3` is going to be rejected. The easiest solution is to uninstall `v2` and reinstall `v3` once values have been adapted to the changes described above. diff --git a/charts/kyverno/templates/admission-controller/deployment.yaml b/charts/kyverno/templates/admission-controller/deployment.yaml index d58a7ec331..202e51c1af 100644 --- a/charts/kyverno/templates/admission-controller/deployment.yaml +++ b/charts/kyverno/templates/admission-controller/deployment.yaml @@ -81,6 +81,7 @@ spec: image: {{ include "kyverno.image" (dict "image" .Values.admissionController.initContainer.image "defaultTag" (default .Chart.AppVersion .Values.admissionController.container.image.tag)) | quote }} imagePullPolicy: {{ default .Values.admissionController.container.image.pullPolicy .Values.admissionController.initContainer.image.pullPolicy }} args: + - --loggingFormat={{ .Values.admissionController.logging.format }} {{- range $key, $value := .Values.admissionController.initContainer.extraArgs }} {{- if $value }} - --{{ $key }}={{ $value }} @@ -119,6 +120,26 @@ spec: imagePullPolicy: {{ .Values.admissionController.container.image.pullPolicy }} args: - --servicePort={{ .Values.admissionController.service.port }} + - --loggingFormat={{ .Values.admissionController.logging.format }} + {{- if .Values.admissionController.tracing.enabled }} + - --enableTracing + - --tracingAddress={{ .Values.admissionController.tracing.address }} + - --tracingPort={{ .Values.admissionController.tracing.port }} + {{- with .Values.admissionController.tracing.creds }} + - --tracingCreds={{ . }} + {{- end }} + {{- end }} + - --disableMetrics={{ .Values.admissionController.metering.disabled }} + {{- if not .Values.admissionController.metering.disabled }} + - --otelConfig={{ .Values.admissionController.metering.config }} + - --metricsPort={{ .Values.admissionController.metering.port }} + {{- with .Values.admissionController.metering.collector }} + - --otelCollector={{ . }} + {{- end }} + {{- with .Values.admissionController.metering.creds }} + - --transportCreds={{ . }} + {{- end }} + {{- end }} {{- if or .Values.imagePullSecrets .Values.existingImagePullSecrets }} - --imagePullSecrets={{- join "," (concat (keys .Values.imagePullSecrets) .Values.existingImagePullSecrets) }} {{- end }} diff --git a/charts/kyverno/values.yaml b/charts/kyverno/values.yaml index e446384f12..d1a8e5e0d5 100644 --- a/charts/kyverno/values.yaml +++ b/charts/kyverno/values.yaml @@ -379,22 +379,6 @@ admissionController: # Cannot be used if `minAvailable` is set. maxUnavailable: - serviceMonitor: - # -- Create a `ServiceMonitor` to collect Prometheus metrics. - enabled: false - # -- Additional labels - additionalLabels: {} - # -- (string) Override namespace - namespace: ~ - # -- Interval to scrape metrics - interval: 30s - # -- Timeout if metrics can't be retrieved in given time interval - scrapeTimeout: 25s - # -- Is TLS required for endpoint - secure: false - # -- TLS Configuration for endpoint - tlsConfig: {} - # -- A writable volume to use for the TUF root initialization. tufRootMountPath: /.sigstore @@ -443,8 +427,7 @@ admissionController: type: RuntimeDefault # -- Additional container args. - extraArgs: - loggingFormat: text + extraArgs: {} # -- Additional container environment variables. extraEnvVars: [] @@ -484,8 +467,7 @@ admissionController: type: RuntimeDefault # -- Additional container args. - extraArgs: - loggingFormat: text + extraArgs: {} # -- Additional container environment variables. extraEnvVars: [] @@ -528,14 +510,54 @@ admissionController: annotations: {} networkPolicy: - # -- When true, use a NetworkPolicy to allow ingress to the webhook # This is useful on clusters using Calico and/or native k8s network policies in a default-deny setup. enabled: false - # -- A list of valid from selectors according to https://kubernetes.io/docs/concepts/services-networking/network-policies. ingressFrom: [] + serviceMonitor: + # -- Create a `ServiceMonitor` to collect Prometheus metrics. + enabled: false + # -- Additional labels + additionalLabels: {} + # -- (string) Override namespace + namespace: ~ + # -- Interval to scrape metrics + interval: 30s + # -- Timeout if metrics can't be retrieved in given time interval + scrapeTimeout: 25s + # -- Is TLS required for endpoint + secure: false + # -- TLS Configuration for endpoint + tlsConfig: {} + + tracing: + # -- Enable tracing + enabled: false + # -- Traces receiver address + address: + # -- Traces receiver port + port: + # -- Traces receiver credentials + creds: '' + + logging: + # -- Logging format + format: text + + metering: + # -- Disable metrics export + disabled: false + # -- Otel configuration, can be `prometheus` or `grpc` + config: prometheus + # -- Prometheus endpoint port + port: 8000 + # -- Otel collector endpoint + collector: '' + # -- Otel collector credentials + creds: '' + # Cleanup controller configuration cleanupController: diff --git a/scripts/config/dev/kyverno.yaml b/scripts/config/dev/kyverno.yaml index c85fa25069..634a6a6905 100644 --- a/scripts/config/dev/kyverno.yaml +++ b/scripts/config/dev/kyverno.yaml @@ -2,20 +2,20 @@ grafana: enabled: true admissionController: + container: + extraArgs: + enablePolicyException: true + serviceMonitor: enabled: true - initContainer: - extraArgs: - loggingFormat: json + tracing: + enabled: true + address: tempo.monitoring + port: 4317 - container: - extraArgs: - loggingFormat: json - enableTracing: true - tracingAddress: tempo.monitoring - tracingPort: 4317 - enablePolicyException: true + logging: + format: json backgroundController: serviceMonitor: diff --git a/scripts/config/standard/kyverno.yaml b/scripts/config/standard/kyverno.yaml index 7f1da34e55..2830a01e72 100644 --- a/scripts/config/standard/kyverno.yaml +++ b/scripts/config/standard/kyverno.yaml @@ -1,12 +1,6 @@ admissionController: - - initContainer: - extraArgs: - loggingFormat: json - container: extraArgs: - loggingFormat: json enablePolicyException: true backgroundController: