From f1491fe6d3e813682c4efaafd29e0f6a16e2f94a Mon Sep 17 00:00:00 2001 From: treydock Date: Thu, 10 Jun 2021 16:53:29 -0400 Subject: [PATCH] Allow metrics service annotations to be defined separate from main service (#1988) * Allow metrics service annotations to be defined separate from main service Signed-off-by: Trey Dockendorf * Add test for metrics during Helm deployment testing Signed-off-by: Trey Dockendorf * Make services separate for kustomize Signed-off-by: Trey Dockendorf * Run 'make kustomize-crd' Signed-off-by: Trey Dockendorf * Fix e2e tests for metrics Signed-off-by: Trey Dockendorf * Fix Helm chart for metrics service Signed-off-by: Trey Dockendorf * Fix helm chart testing Signed-off-by: Trey Dockendorf --- .github/workflows/e2e.yaml | 2 +- charts/kyverno/templates/service.yaml | 30 ++++++++++++++++++------ charts/kyverno/templates/tests/test.yaml | 8 +++++++ charts/kyverno/values.yaml | 9 +++++-- definitions/install.yaml | 19 +++++++++++++++ definitions/install_debug.yaml | 13 ++++++++++ definitions/k8s-resource/service.yaml | 14 +++++++++++ 7 files changed, 85 insertions(+), 10 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index cff6f55e96..ac4e5fdb04 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -84,7 +84,7 @@ jobs: sleep 20 echo ">>> Expose the Kyverno's service's metric server to the host" - kubectl port-forward svc/kyverno-svc -n kyverno 8000:8000 & + kubectl port-forward svc/kyverno-svc-metrics -n kyverno 8000:8000 & echo ">>> Run Kyverno e2e test" make test-e2e kubectl delete -f ${GITHUB_WORKSPACE}/definitions/install.yaml diff --git a/charts/kyverno/templates/service.yaml b/charts/kyverno/templates/service.yaml index caec996323..515e72a474 100644 --- a/charts/kyverno/templates/service.yaml +++ b/charts/kyverno/templates/service.yaml @@ -17,13 +17,29 @@ spec: {{- if and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort)) }} nodePort: {{ .Values.service.nodePort }} {{- end }} - - port: {{ .Values.service.metricsPort }} - targetPort: 8000 - protocol: TCP - name: metrics-port - {{- if and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort)) }} - nodePort: {{ .Values.service.metricsNodePort }} - {{- end }} selector: {{ include "kyverno.matchLabels" . | nindent 4 }} app: kyverno type: {{ .Values.service.type }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kyverno.serviceName" . }}-metrics + labels: {{ include "kyverno.labels" . | nindent 4 }} + app: kyverno + namespace: {{ template "kyverno.namespace" . }} + {{- with .Values.metricsService.annotations }} + annotations: {{ tpl (toYaml .) $ | nindent 4 }} + {{- end }} +spec: + ports: + - port: {{ .Values.metricsService.port }} + targetPort: 8000 + protocol: TCP + name: metrics-port + {{- if and (eq .Values.metricsService.type "NodePort") (not (empty .Values.metricsService.nodePort)) }} + nodePort: {{ .Values.metricsService.metricsNodePort }} + {{- end }} + selector: {{ include "kyverno.matchLabels" . | nindent 4 }} + app: kyverno + type: {{ .Values.metricsService.type }} diff --git a/charts/kyverno/templates/tests/test.yaml b/charts/kyverno/templates/tests/test.yaml index f176cdd475..a7493ce097 100644 --- a/charts/kyverno/templates/tests/test.yaml +++ b/charts/kyverno/templates/tests/test.yaml @@ -7,6 +7,7 @@ metadata: annotations: "helm.sh/hook": test spec: + restartPolicy: Never containers: - name: wget image: busybox @@ -15,4 +16,11 @@ spec: - -c - | sleep 20 ; wget -O- -S --no-check-certificate https://{{ template "kyverno.serviceName" . }}:{{ .Values.service.port }}/health/liveness + - name: wget-metrics + image: busybox + command: + - /bin/sh + - -c + - | + sleep 20 ; wget -O- -S --no-check-certificate http://{{ template "kyverno.serviceName" . }}-metrics:{{ .Values.metricsService.port }}/metrics restartPolicy: Never diff --git a/charts/kyverno/values.yaml b/charts/kyverno/values.yaml index 32c96d4bd2..8cae6ef37b 100644 --- a/charts/kyverno/values.yaml +++ b/charts/kyverno/values.yaml @@ -148,10 +148,15 @@ service: type: ClusterIP # Only used if service.type is NodePort nodePort: + annotations: {} + +metricsService: + create: true + type: ClusterIP ## Kyverno's metrics server will be exposed at this port - metricsPort: 8000 + port: 8000 ## The Node's port which will allow access Kyverno's metrics at the host level. Only used if service.type is NodePort. - metricsNodePort: 8000 + nodePort: ## Provide any additional annotations which may be required. This can be used to ## set the LoadBalancer service type to internal only. ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer diff --git a/definitions/install.yaml b/definitions/install.yaml index 4c9d9eb2e0..1d1df7046b 100644 --- a/definitions/install.yaml +++ b/definitions/install.yaml @@ -2629,6 +2629,25 @@ spec: - name: https port: 443 targetPort: https + selector: + app: kyverno + app.kubernetes.io/name: kyverno +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: kyverno + app.kubernetes.io/component: kyverno + app.kubernetes.io/instance: kyverno + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: kyverno + app.kubernetes.io/part-of: kyverno + app.kubernetes.io/version: v1.4.0-rc1 + name: kyverno-svc-metrics + namespace: kyverno +spec: + ports: - name: metrics-port port: 8000 targetPort: metrics-port diff --git a/definitions/install_debug.yaml b/definitions/install_debug.yaml index e84b190b14..798b3cebd8 100755 --- a/definitions/install_debug.yaml +++ b/definitions/install_debug.yaml @@ -2466,6 +2466,19 @@ spec: - name: https port: 443 targetPort: https + selector: + app: kyverno + app.kubernetes.io/name: kyverno +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: kyverno + name: kyverno-svc-metrics + namespace: kyverno +spec: + ports: - name: metrics-port port: 8000 targetPort: metrics-port diff --git a/definitions/k8s-resource/service.yaml b/definitions/k8s-resource/service.yaml index e0e4965911..a4d5e00dde 100644 --- a/definitions/k8s-resource/service.yaml +++ b/definitions/k8s-resource/service.yaml @@ -11,6 +11,20 @@ spec: - port: 443 name: https targetPort: https + selector: + app: kyverno + # do not remove + app.kubernetes.io/name: kyverno +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: kyverno + namespace: kyverno + name: kyverno-svc-metrics +spec: + ports: - port: 8000 name: metrics-port targetPort: metrics-port