Allow metrics service annotations to be defined separate from main service (#1988)

* Allow metrics service annotations to be defined separate from main service

Signed-off-by: Trey Dockendorf <tdockendorf@osc.edu>

* Add test for metrics during Helm deployment testing

Signed-off-by: Trey Dockendorf <tdockendorf@osc.edu>

* Make services separate for kustomize

Signed-off-by: Trey Dockendorf <tdockendorf@osc.edu>

* Run 'make kustomize-crd'

Signed-off-by: Trey Dockendorf <tdockendorf@osc.edu>

* Fix e2e tests for metrics

Signed-off-by: Trey Dockendorf <tdockendorf@osc.edu>

* Fix Helm chart for metrics service

Signed-off-by: Trey Dockendorf <tdockendorf@osc.edu>

* Fix helm chart testing

Signed-off-by: Trey Dockendorf <tdockendorf@osc.edu>

.github/workflows/e2e.yaml

@@ -84,7 +84,7 @@ jobs:
           sleep 20
           
           echo ">>> Expose the Kyverno's service's metric server to the host"
-          kubectl port-forward svc/kyverno-svc -n kyverno 8000:8000 &
+          kubectl port-forward svc/kyverno-svc-metrics -n kyverno 8000:8000 &
           echo ">>> Run Kyverno e2e test"
           make test-e2e
           kubectl delete -f ${GITHUB_WORKSPACE}/definitions/install.yaml

charts/kyverno/templates/service.yaml

@@ -17,13 +17,29 @@ spec:
     {{- if and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort)) }}
     nodePort: {{ .Values.service.nodePort }}
     {{- end }}
-  - port: {{ .Values.service.metricsPort }}
-    targetPort: 8000
-    protocol: TCP
-    name: metrics-port
-    {{- if and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort)) }}
-    nodePort: {{ .Values.service.metricsNodePort }}
-    {{- end }}
   selector: {{ include "kyverno.matchLabels" . | nindent 4 }}
     app: kyverno
   type: {{ .Values.service.type }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "kyverno.serviceName" . }}-metrics
+  labels: {{ include "kyverno.labels" . | nindent 4 }}
+    app: kyverno
+  namespace: {{ template "kyverno.namespace" . }}
+  {{- with .Values.metricsService.annotations }}
+  annotations: {{ tpl (toYaml .) $ | nindent 4 }}
+  {{- end }}
+spec:
+  ports:
+  - port: {{ .Values.metricsService.port }}
+    targetPort: 8000
+    protocol: TCP
+    name: metrics-port
+    {{- if and (eq .Values.metricsService.type "NodePort") (not (empty .Values.metricsService.nodePort)) }}
+    nodePort: {{ .Values.metricsService.metricsNodePort }}
+    {{- end }}
+  selector: {{ include "kyverno.matchLabels" . | nindent 4 }}
+    app: kyverno
+  type: {{ .Values.metricsService.type }}

charts/kyverno/templates/tests/test.yaml

@@ -7,6 +7,7 @@ metadata:
   annotations:
     "helm.sh/hook": test
 spec:
+  restartPolicy: Never
   containers:
     - name: wget
       image: busybox
@@ -15,4 +16,11 @@ spec:
         - -c
         - |
           sleep 20 ; wget -O- -S --no-check-certificate https://{{ template "kyverno.serviceName" . }}:{{ .Values.service.port }}/health/liveness
+    - name: wget-metrics
+      image: busybox
+      command:
+        - /bin/sh
+        - -c
+        - |
+          sleep 20 ; wget -O- -S --no-check-certificate http://{{ template "kyverno.serviceName" . }}-metrics:{{ .Values.metricsService.port }}/metrics
   restartPolicy: Never

charts/kyverno/values.yaml

@@ -148,10 +148,15 @@ service:
   type: ClusterIP
   # Only used if service.type is NodePort
   nodePort:
+  annotations: {}
+
+metricsService:
+  create: true
+  type: ClusterIP
   ## Kyverno's metrics server will be exposed at this port
-  metricsPort: 8000
+  port: 8000
   ## The Node's port which will allow access Kyverno's metrics at the host level. Only used if service.type is NodePort.
-  metricsNodePort: 8000
+  nodePort:
   ## Provide any additional annotations which may be required. This can be used to
   ## set the LoadBalancer service type to internal only.
   ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer

definitions/install.yaml

@@ -2629,6 +2629,25 @@ spec:
   - name: https
     port: 443
     targetPort: https
+  selector:
+    app: kyverno
+    app.kubernetes.io/name: kyverno
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: kyverno
+    app.kubernetes.io/component: kyverno
+    app.kubernetes.io/instance: kyverno
+    app.kubernetes.io/managed-by: Kustomize
+    app.kubernetes.io/name: kyverno
+    app.kubernetes.io/part-of: kyverno
+    app.kubernetes.io/version: v1.4.0-rc1
+  name: kyverno-svc-metrics
+  namespace: kyverno
+spec:
+  ports:
   - name: metrics-port
     port: 8000
     targetPort: metrics-port

definitions/install_debug.yaml

@@ -2466,6 +2466,19 @@ spec:
   - name: https
     port: 443
     targetPort: https
+  selector:
+    app: kyverno
+    app.kubernetes.io/name: kyverno
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: kyverno
+  name: kyverno-svc-metrics
+  namespace: kyverno
+spec:
+  ports:
   - name: metrics-port
     port: 8000
     targetPort: metrics-port

definitions/k8s-resource/service.yaml

@@ -11,6 +11,20 @@ spec:
   - port: 443
     name: https
     targetPort: https
+  selector:
+    app: kyverno
+    # do not remove
+    app.kubernetes.io/name: kyverno
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: kyverno
+  namespace: kyverno
+  name: kyverno-svc-metrics
+spec:
+  ports:
   - port: 8000
     name: metrics-port
     targetPort: metrics-port