refactor: add config support to webhook controller (#4838)

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

Co-authored-by: Prateek Pandey <prateek.pandey@nirmata.com>

cmd/kyverno/main.go

@@ -449,6 +449,7 @@ func createrLeaderControllers(
 			kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(),
 		),
 		kubeKyvernoInformer.Core().V1().Secrets(),
+		kubeKyvernoInformer.Core().V1().ConfigMaps(),
 		kubeInformer.Admissionregistration().V1().MutatingWebhookConfigurations(),
 		kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(),
 	)

pkg/config/config.go

@@ -143,12 +143,17 @@ type configuration struct {
 }
 
 // NewConfiguration ...
-func NewConfiguration(client kubernetes.Interface, updateWebhookConfigurations chan<- bool) (Configuration, error) {
+func NewDefaultConfiguration(updateWebhookConfigurations chan<- bool) *configuration {
-	cd := &configuration{
+	return &configuration{
 		updateWebhookConfigurations: updateWebhookConfigurations,
 		restrictDevelopmentUsername: []string{"minikube-user", "kubernetes-admin"},
 		excludeGroupRole:            defaultExcludeGroupRole,
 	}
+}
+
+// NewConfiguration ...
+func NewConfiguration(client kubernetes.Interface, updateWebhookConfigurations chan<- bool) (Configuration, error) {
+	cd := NewDefaultConfiguration(updateWebhookConfigurations)
 	if cm, err := client.CoreV1().ConfigMaps(kyvernoNamespace).Get(context.TODO(), kyvernoConfigMapName, metav1.GetOptions{}); err != nil {
 		if !errors.IsNotFound(err) {
 			return nil, err
@@ -227,7 +232,9 @@ func (cd *configuration) Load(cm *corev1.ConfigMap) {
 	}
 	if updateWebhook {
 		logger.Info("webhook configurations changed, updating webhook configurations")
-		cd.updateWebhookConfigurations <- true
+		if cd.updateWebhookConfigurations != nil {
+			cd.updateWebhookConfigurations <- true
+		}
 	}
 }
 

pkg/controllers/webhook/controller.go

@@ -35,9 +35,10 @@ type controller struct {
 	vwcClient    controllerutils.UpdateClient[*admissionregistrationv1.ValidatingWebhookConfiguration]
 
 	// listers
-	secretLister corev1listers.SecretLister
+	secretLister    corev1listers.SecretLister
-	mwcLister    admissionregistrationv1listers.MutatingWebhookConfigurationLister
+	configMapLister corev1listers.ConfigMapLister
-	vwcLister    admissionregistrationv1listers.ValidatingWebhookConfigurationLister
+	mwcLister       admissionregistrationv1listers.MutatingWebhookConfigurationLister
+	vwcLister       admissionregistrationv1listers.ValidatingWebhookConfigurationLister
 
 	// queue
 	queue      workqueue.RateLimitingInterface
@@ -50,38 +51,34 @@ func NewController(
 	mwcClient controllerutils.UpdateClient[*admissionregistrationv1.MutatingWebhookConfiguration],
 	vwcClient controllerutils.UpdateClient[*admissionregistrationv1.ValidatingWebhookConfiguration],
 	secretInformer corev1informers.SecretInformer,
+	configMapInformer corev1informers.ConfigMapInformer,
 	mwcInformer admissionregistrationv1informers.MutatingWebhookConfigurationInformer,
 	vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer,
 ) controllers.Controller {
 	queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ControllerName)
 	c := controller{
-		secretClient: secretClient,
+		secretClient:    secretClient,
-		mwcClient:    mwcClient,
+		mwcClient:       mwcClient,
-		vwcClient:    vwcClient,
+		vwcClient:       vwcClient,
-		secretLister: secretInformer.Lister(),
+		secretLister:    secretInformer.Lister(),
-		mwcLister:    mwcInformer.Lister(),
+		configMapLister: configMapInformer.Lister(),
-		vwcLister:    vwcInformer.Lister(),
+		mwcLister:       mwcInformer.Lister(),
-		queue:        queue,
+		vwcLister:       vwcInformer.Lister(),
-		mwcEnqueue:   controllerutils.AddDefaultEventHandlers(logger.V(3), mwcInformer.Informer(), queue),
+		queue:           queue,
-		vwcEnqueue:   controllerutils.AddDefaultEventHandlers(logger.V(3), vwcInformer.Informer(), queue),
+		mwcEnqueue:      controllerutils.AddDefaultEventHandlers(logger.V(3), mwcInformer.Informer(), queue),
+		vwcEnqueue:      controllerutils.AddDefaultEventHandlers(logger.V(3), vwcInformer.Informer(), queue),
 	}
-	controllerutils.AddEventHandlers(
+	controllerutils.AddEventHandlersT(
 		secretInformer.Informer(),
-		func(obj interface{}) {
+		func(obj *corev1.Secret) { c.secretChanged(obj) },
-			if err := c.enqueue(obj.(*corev1.Secret)); err != nil {
+		func(_, obj *corev1.Secret) { c.secretChanged(obj) },
-				logger.Error(err, "failed to enqueue")
+		func(obj *corev1.Secret) { c.secretChanged(obj) },
-			}
+	)
-		},
+	controllerutils.AddEventHandlersT(
-		func(_, obj interface{}) {
+		configMapInformer.Informer(),
-			if err := c.enqueue(obj.(*corev1.Secret)); err != nil {
+		func(obj *corev1.ConfigMap) { c.configMapChanged(obj) },
-				logger.Error(err, "failed to enqueue")
+		func(_, obj *corev1.ConfigMap) { c.configMapChanged(obj) },
-			}
+		func(obj *corev1.ConfigMap) { c.configMapChanged(obj) },
-		},
-		func(obj interface{}) {
-			if err := c.enqueue(obj.(*corev1.Secret)); err != nil {
-				logger.Error(err, "failed to enqueue")
-			}
-		},
 	)
 	return &c
 }
@@ -90,38 +87,60 @@ func (c *controller) Run(ctx context.Context, workers int) {
 	controllerutils.Run(ctx, ControllerName, logger.V(3), c.queue, workers, maxRetries, c.reconcile)
 }
 
-func (c *controller) enqueue(obj *corev1.Secret) error {
+func (c *controller) secretChanged(secret *corev1.Secret) {
-	if obj.GetName() == tls.GenerateRootCASecretName() && obj.GetNamespace() == config.KyvernoNamespace() {
+	if secret.GetName() == tls.GenerateRootCASecretName() && secret.GetNamespace() == config.KyvernoNamespace() {
-		requirement, err := labels.NewRequirement("webhook.kyverno.io/managed-by", selection.Equals, []string{kyvernov1.ValueKyvernoApp})
+		if err := c.enqueueAll(); err != nil {
-		if err != nil {
+			logger.Error(err, "failed to enqueue on secret change")
-			// TODO: log error
-			return err
 		}
-		selector := labels.Everything().Add(*requirement)
+	}
-		mwcs, err := c.mwcLister.List(selector)
+}
+
+func (c *controller) configMapChanged(cm *corev1.ConfigMap) {
+	if cm.GetName() == config.KyvernoConfigMapName() && cm.GetNamespace() == config.KyvernoNamespace() {
+		if err := c.enqueueAll(); err != nil {
+			logger.Error(err, "failed to enqueue on configmap change")
+		}
+	}
+}
+
+func (c *controller) enqueueAll() error {
+	requirement, err := labels.NewRequirement("webhook.kyverno.io/managed-by", selection.Equals, []string{kyvernov1.ValueKyvernoApp})
+	if err != nil {
+		return err
+	}
+	selector := labels.Everything().Add(*requirement)
+	mwcs, err := c.mwcLister.List(selector)
+	if err != nil {
+		return err
+	}
+	for _, mwc := range mwcs {
+		err = c.mwcEnqueue(mwc)
 		if err != nil {
 			return err
 		}
-		for _, mwc := range mwcs {
+	}
-			err = c.mwcEnqueue(mwc)
+	vwcs, err := c.vwcLister.List(selector)
-			if err != nil {
+	if err != nil {
-				return err
+		return err
-			}
+	}
-		}
+	for _, vwc := range vwcs {
-		vwcs, err := c.vwcLister.List(selector)
+		err = c.vwcEnqueue(vwc)
 		if err != nil {
 			return err
 		}
-		for _, vwc := range vwcs {
-			err = c.vwcEnqueue(vwc)
-			if err != nil {
-				return err
-			}
-		}
 	}
 	return nil
 }
 
+func (c *controller) loadConfig() config.Configuration {
+	cfg := config.NewDefaultConfiguration(nil)
+	cm, err := c.configMapLister.ConfigMaps(config.KyvernoNamespace()).Get(config.KyvernoConfigMapName())
+	if err == nil {
+		cfg.Load(cm)
+	}
+	return cfg
+}
+
 func (c *controller) reconcileMutatingWebhookConfiguration(ctx context.Context, logger logr.Logger, name string) error {
 	w, err := c.mwcLister.Get(name)
 	if err != nil {
@@ -134,13 +153,21 @@ func (c *controller) reconcileMutatingWebhookConfiguration(ctx context.Context,
 	if labels == nil || labels["webhook.kyverno.io/managed-by"] != kyvernov1.ValueKyvernoApp {
 		return nil
 	}
+	cfg := c.loadConfig()
+	webhookCfg := config.WebhookConfig{}
+	webhookCfgs := cfg.GetWebhooks()
+	if len(webhookCfgs) > 0 {
+		webhookCfg = webhookCfgs[0]
+	}
+	caData, err := tls.ReadRootCASecret(c.secretClient)
+	if err != nil {
+		return err
+	}
 	_, err = controllerutils.Update(ctx, w, c.mwcClient, func(w *admissionregistrationv1.MutatingWebhookConfiguration) error {
-		caData, err := tls.ReadRootCASecret(c.secretClient)
-		if err != nil {
-			return err
-		}
 		for i := range w.Webhooks {
 			w.Webhooks[i].ClientConfig.CABundle = caData
+			w.Webhooks[i].ObjectSelector = webhookCfg.ObjectSelector
+			w.Webhooks[i].NamespaceSelector = webhookCfg.NamespaceSelector
 		}
 		return nil
 	})
@@ -159,13 +186,21 @@ func (c *controller) reconcileValidatingWebhookConfiguration(ctx context.Context
 	if labels == nil || labels["webhook.kyverno.io/managed-by"] != kyvernov1.ValueKyvernoApp {
 		return nil
 	}
+	cfg := c.loadConfig()
+	webhookCfg := config.WebhookConfig{}
+	webhookCfgs := cfg.GetWebhooks()
+	if len(webhookCfgs) > 0 {
+		webhookCfg = webhookCfgs[0]
+	}
+	caData, err := tls.ReadRootCASecret(c.secretClient)
+	if err != nil {
+		return err
+	}
 	_, err = controllerutils.Update(ctx, w, c.vwcClient, func(w *admissionregistrationv1.ValidatingWebhookConfiguration) error {
-		caData, err := tls.ReadRootCASecret(c.secretClient)
-		if err != nil {
-			return err
-		}
 		for i := range w.Webhooks {
 			w.Webhooks[i].ClientConfig.CABundle = caData
+			w.Webhooks[i].ObjectSelector = webhookCfg.ObjectSelector
+			w.Webhooks[i].NamespaceSelector = webhookCfg.NamespaceSelector
 		}
 		return nil
 	})

pkg/utils/controller/handlers.go

@@ -10,11 +10,15 @@ import (
 )
 
 type (
-	addFunc     func(interface{})
+	addFunc             func(interface{})
-	updateFunc  func(interface{}, interface{})
+	updateFunc          func(interface{}, interface{})
-	deleteFunc  func(interface{})
+	deleteFunc          func(interface{})
-	keyFunc     func(interface{}) (interface{}, error)
+	addFuncT[T any]     func(T)
-	EnqueueFunc func(interface{}) error
+	updateFuncT[T any]  func(T, T)
+	deleteFuncT[T any]  func(T)
+	keyFunc             func(interface{}) (interface{}, error)
+	EnqueueFunc         func(interface{}) error
+	EnqueueFuncT[T any] func(T) error
 )
 
 func AddEventHandlers(informer cache.SharedInformer, a addFunc, u updateFunc, d deleteFunc) {
@@ -25,6 +29,14 @@ func AddEventHandlers(informer cache.SharedInformer, a addFunc, u updateFunc, d
 	})
 }
 
+func AddEventHandlersT[T any](informer cache.SharedInformer, a addFuncT[T], u updateFuncT[T], d deleteFuncT[T]) {
+	informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    func(obj interface{}) { a(obj.(T)) },
+		UpdateFunc: func(old, obj interface{}) { u(old.(T), obj.(T)) },
+		DeleteFunc: func(obj interface{}) { d(obj.(T)) },
+	})
+}
+
 func AddKeyedEventHandlers(logger logr.Logger, informer cache.SharedInformer, queue workqueue.RateLimitingInterface, parseKey keyFunc) EnqueueFunc {
 	enqueueFunc := LogError(logger, Parse(parseKey, Queue(queue)))
 	AddEventHandlers(informer, AddFunc(logger, enqueueFunc), UpdateFunc(logger, enqueueFunc), DeleteFunc(logger, enqueueFunc))