feat: add certs controller to cleanup policies (#5671)

* feat: add certs controller to cleanup policies

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* webhook controller

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

charts/kyverno/README.md

@@ -219,6 +219,7 @@ The command removes all the Kubernetes components associated with the chart and
 | cleanupController.rbac.create | bool | `true` | Create RBAC resources |
 | cleanupController.rbac.serviceAccount.name | string | `nil` | Service account name |
 | cleanupController.rbac.clusterRole.extraResources | list | `[]` | Extra resource permissions to add in the cluster role |
+| cleanupController.createSelfSignedCert | bool | `false` | Create self-signed certificates at deployment time. The certificates won't be automatically renewed if this is set to `true`. |
 | cleanupController.image.registry | string | `nil` | Image registry |
 | cleanupController.image.repository | string | `"ghcr.io/kyverno/cleanup-controller"` | Image repository |
 | cleanupController.image.tag | string | `nil` | Image tag Defaults to appVersion in Chart.yaml if omitted |

charts/kyverno/templates/cleanup-controller/clusterrole.yaml

@@ -7,6 +7,17 @@ metadata:
   labels:
     {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
 rules:
+  - apiGroups:
+      - admissionregistration.k8s.io
+    resources:
+      - validatingwebhookconfigurations
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - update
+      - watch
   - apiGroups:
       - ''
     resources:

charts/kyverno/templates/cleanup-controller/role.yaml

@@ -12,6 +12,15 @@ rules:
     - ''
   resources:
     - secrets
+  verbs:
+    - get
+    - list
+    - watch
+    - create
+    - update
+- apiGroups:
+    - ''
+  resources:
     - configmaps
   verbs:
     - get

charts/kyverno/templates/cleanup-controller/secret.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.cleanupController.enabled -}}
+{{- if .Values.cleanupController.createSelfSignedCert -}}
 {{- $ca := genCA (printf "*.%s.svc" (include "kyverno.namespace" .)) 1024 -}}
 {{- $svcName := (printf "%s.%s.svc" (include "kyverno.cleanup-controller.name" .) (include "kyverno.namespace" .)) -}}
 {{- $cert := genSignedCert $svcName nil (list $svcName) 1024 $ca -}}
@@ -6,9 +7,9 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: {{ template "kyverno.cleanup-controller.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-ca
+  namespace: {{ template "kyverno.namespace" . }}
   labels:
     {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
-  namespace: {{ template "kyverno.namespace" . }}
 type: kubernetes.io/tls
 data:
   tls.key: {{ $ca.Key | b64enc }}
@@ -18,45 +19,14 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: {{ template "kyverno.cleanup-controller.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-pair
+  namespace: {{ template "kyverno.namespace" . }}
   labels:
     {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
-  namespace: {{ template "kyverno.namespace" . }}
+  annotations:
+    self-signed-cert: "true"
 type: kubernetes.io/tls
 data:
   tls.key: {{ $cert.Key | b64enc }}
   tls.crt: {{ $cert.Cert | b64enc }}
----
-apiVersion: admissionregistration.k8s.io/v1
-kind: ValidatingWebhookConfiguration
-metadata:
-  name: {{ template "kyverno.cleanup-controller.name" . }}
-  labels:
-    {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
-webhooks:
-- admissionReviewVersions:
-  - v1beta1
-  clientConfig:
-    caBundle:  {{ $ca.Cert | b64enc }}
-    service:
-      name: {{ template "kyverno.cleanup-controller.name" . }}
-      namespace: {{ template "kyverno.namespace" . }}
-      path: /validate
-      port: 443
-  failurePolicy: Fail
-  matchPolicy: Equivalent
-  name: {{ printf "%s.%s.svc" (include "kyverno.cleanup-controller.name" .) (include "kyverno.namespace" .) }}
-  rules:
-  - apiGroups:
-    - kyverno.io
-    apiVersions:
-    - v2alpha1
-    operations:
-    - CREATE
-    - UPDATE
-    resources:
-    - clustercleanuppolicies/*
-    - cleanuppolicies/*
-    scope: '*'
-  sideEffects: None
-  timeoutSeconds: 10
+{{- end -}}
 {{- end -}}

charts/kyverno/templates/secret.yaml

@@ -6,6 +6,7 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: {{ template "kyverno.serviceName" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-ca
+  namespace: {{ template "kyverno.namespace" . }}
   labels: {{ include "kyverno.labels" . | nindent 4 }}
     app: kyverno
 type: kubernetes.io/tls
@@ -17,6 +18,7 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: {{ template "kyverno.serviceName" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-pair
+  namespace: {{ template "kyverno.namespace" . }}
   labels: {{ include "kyverno.labels" . | nindent 4 }}
     app: kyverno
   annotations:

charts/kyverno/values.yaml

@@ -506,6 +506,10 @@ cleanupController:
       #   resources:
       #     - pods
 
+  # -- Create self-signed certificates at deployment time.
+  # The certificates won't be automatically renewed if this is set to `true`.
+  createSelfSignedCert: false
+
   image:
     # -- Image registry
     registry:

cmd/cleanup-controller/controller.go

@@ -0,0 +1,190 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/go-logr/logr"
+	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
+	"github.com/kyverno/kyverno/pkg/config"
+	"github.com/kyverno/kyverno/pkg/controllers"
+	"github.com/kyverno/kyverno/pkg/logging"
+	"github.com/kyverno/kyverno/pkg/tls"
+	controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	admissionregistrationv1informers "k8s.io/client-go/informers/admissionregistration/v1"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	admissionregistrationv1listers "k8s.io/client-go/listers/admissionregistration/v1"
+	corev1listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/util/workqueue"
+)
+
+const (
+	// Workers is the number of workers for this controller
+	Workers        = 2
+	ControllerName = "webhook-controller"
+	maxRetries     = 10
+	managedByLabel = "webhook.kyverno.io/managed-by"
+)
+
+var (
+	none = admissionregistrationv1.SideEffectClassNone
+	fail = admissionregistrationv1.Fail
+)
+
+var logger = logging.ControllerLogger(ControllerName)
+
+type controller struct {
+	// clients
+	vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration]
+
+	// listers
+	vwcLister    admissionregistrationv1listers.ValidatingWebhookConfigurationLister
+	secretLister corev1listers.SecretNamespaceLister
+
+	// queue
+	queue workqueue.RateLimitingInterface
+
+	// config
+	webhookName string
+	server      string
+}
+
+func NewController(
+	vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration],
+	vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer,
+	secretInformer corev1informers.SecretInformer,
+	webhookName string,
+	server string,
+) controllers.Controller {
+	queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ControllerName)
+	c := controller{
+		vwcClient:    vwcClient,
+		vwcLister:    vwcInformer.Lister(),
+		secretLister: secretInformer.Lister().Secrets(config.KyvernoNamespace()),
+		queue:        queue,
+		webhookName:  webhookName,
+		server:       server,
+	}
+	controllerutils.AddDefaultEventHandlers(logger, vwcInformer.Informer(), queue)
+	controllerutils.AddEventHandlersT(
+		secretInformer.Informer(),
+		func(obj *corev1.Secret) {
+			if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == tls.GenerateRootCASecretName() {
+				c.enqueue()
+			}
+		},
+		func(_, obj *corev1.Secret) {
+			if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == tls.GenerateRootCASecretName() {
+				c.enqueue()
+			}
+		},
+		func(obj *corev1.Secret) {
+			if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == tls.GenerateRootCASecretName() {
+				c.enqueue()
+			}
+		},
+	)
+	return &c
+}
+
+func (c *controller) Run(ctx context.Context, workers int) {
+	c.enqueue()
+	controllerutils.Run(ctx, logger, ControllerName, time.Second, c.queue, workers, maxRetries, c.reconcile)
+}
+
+func (c *controller) enqueue() {
+	c.queue.Add(c.webhookName)
+}
+
+func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, _, _ string) error {
+	if key != c.webhookName {
+		return nil
+	}
+	caData, err := tls.ReadRootCASecret(c.secretLister)
+	if err != nil {
+		return err
+	}
+	desired, err := c.build(caData)
+	if err != nil {
+		return err
+	}
+	observed, err := c.vwcLister.Get(desired.Name)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			_, err := c.vwcClient.Create(ctx, desired, metav1.CreateOptions{})
+			return err
+		}
+		return err
+	}
+	_, err = controllerutils.Update(ctx, observed, c.vwcClient, func(w *admissionregistrationv1.ValidatingWebhookConfiguration) error {
+		w.Labels = desired.Labels
+		w.OwnerReferences = desired.OwnerReferences
+		w.Webhooks = desired.Webhooks
+		return nil
+	})
+	return err
+}
+
+func objectMeta(name string, owner ...metav1.OwnerReference) metav1.ObjectMeta {
+	return metav1.ObjectMeta{
+		Name: name,
+		Labels: map[string]string{
+			managedByLabel: kyvernov1.ValueKyvernoApp,
+		},
+		OwnerReferences: owner,
+	}
+}
+
+func (c *controller) build(caBundle []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) {
+	return &admissionregistrationv1.ValidatingWebhookConfiguration{
+			ObjectMeta: objectMeta(c.webhookName),
+			Webhooks: []admissionregistrationv1.ValidatingWebhook{{
+				Name:         fmt.Sprintf("%s.%s.svc", config.KyvernoServiceName(), config.KyvernoNamespace()),
+				ClientConfig: c.clientConfig(caBundle, validatingWebhookServicePath),
+				Rules: []admissionregistrationv1.RuleWithOperations{{
+					Rule: admissionregistrationv1.Rule{
+						APIGroups: []string{
+							"kyverno.io",
+						},
+						APIVersions: []string{
+							"v2alpha1",
+						},
+						Resources: []string{
+							"cleanuppolicies/*",
+							"clustercleanuppolicies/*",
+						},
+					},
+					Operations: []admissionregistrationv1.OperationType{
+						admissionregistrationv1.Create,
+						admissionregistrationv1.Update,
+					},
+				}},
+				FailurePolicy:           &fail,
+				SideEffects:             &none,
+				AdmissionReviewVersions: []string{"v1"},
+			}},
+		},
+		nil
+}
+
+func (c *controller) clientConfig(caBundle []byte, path string) admissionregistrationv1.WebhookClientConfig {
+	clientConfig := admissionregistrationv1.WebhookClientConfig{
+		CABundle: caBundle,
+	}
+	if c.server == "" {
+		clientConfig.Service = &admissionregistrationv1.ServiceReference{
+			Namespace: config.KyvernoNamespace(),
+			Name:      config.KyvernoServiceName(),
+			Path:      &path,
+		}
+	} else {
+		url := fmt.Sprintf("https://%s%s", c.server, path)
+		clientConfig.URL = &url
+	}
+	return clientConfig
+}

cmd/cleanup-controller/main.go

@@ -16,6 +16,7 @@ import (
 	kubeclient "github.com/kyverno/kyverno/pkg/clients/kube"
 	kyvernoclient "github.com/kyverno/kyverno/pkg/clients/kyverno"
 	"github.com/kyverno/kyverno/pkg/config"
+	"github.com/kyverno/kyverno/pkg/controllers/certmanager"
 	"github.com/kyverno/kyverno/pkg/controllers/cleanup"
 	"github.com/kyverno/kyverno/pkg/leaderelection"
 	"github.com/kyverno/kyverno/pkg/metrics"
@@ -32,7 +33,6 @@ const (
 // TODO:
 // - helm review labels / selectors
 // - implement probes
-// - better certs management
 // - supports certs in cronjob
 
 type probes struct{}
@@ -88,8 +88,38 @@ func main() {
 			// informer factories
 			kubeInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, resyncPeriod)
 			kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(kyvernoClient, resyncPeriod)
+			kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace()))
+			// listers
+			secretLister := kubeKyvernoInformer.Core().V1().Secrets().Lister().Secrets(config.KyvernoNamespace())
 			// controllers
-			controller := internal.NewController(
+			renewer := tls.NewCertRenewer(
+				kubeClient.CoreV1().Secrets(config.KyvernoNamespace()),
+				secretLister,
+				tls.CertRenewalInterval,
+				tls.CAValidityDuration,
+				tls.TLSValidityDuration,
+				"",
+			)
+			certController := internal.NewController(
+				certmanager.ControllerName,
+				certmanager.NewController(
+					kubeKyvernoInformer.Core().V1().Secrets(),
+					renewer,
+				),
+				certmanager.Workers,
+			)
+			webhookController := internal.NewController(
+				ControllerName,
+				NewController(
+					kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(),
+					kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(),
+					kubeKyvernoInformer.Core().V1().Secrets(),
+					"kyverno-cleanup-policies",
+					"",
+				),
+				Workers,
+			)
+			cleanupController := internal.NewController(
 				cleanup.ControllerName,
 				cleanup.NewController(
 					kubeClient,
@@ -101,13 +131,15 @@ func main() {
 				cleanup.Workers,
 			)
 			// start informers and wait for cache sync
-			if !internal.StartInformersAndWaitForCacheSync(ctx, kyvernoInformer, kubeInformer) {
+			if !internal.StartInformersAndWaitForCacheSync(ctx, kyvernoInformer, kubeInformer, kubeKyvernoInformer) {
 				logger.Error(errors.New("failed to wait for cache sync"), "failed to wait for cache sync")
 				os.Exit(1)
 			}
 			// start leader controllers
 			var wg sync.WaitGroup
-			controller.Run(ctx, logger.WithName("cleanup-controller"), &wg)
+			certController.Run(ctx, logger, &wg)
+			webhookController.Run(ctx, logger, &wg)
+			cleanupController.Run(ctx, logger, &wg)
 			// wait all controllers shut down
 			wg.Wait()
 		},