From 7bfcf7d7e226a25b824eb365c23aae3c6efce74c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Fri, 7 Oct 2022 13:32:38 +0200 Subject: [PATCH] refactor: add config support to webhook controller (#4838) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Charles-Edouard Brétéché Co-authored-by: Prateek Pandey --- cmd/kyverno/main.go | 1 + pkg/config/config.go | 13 ++- pkg/controllers/webhook/controller.go | 149 ++++++++++++++++---------- pkg/utils/controller/handlers.go | 22 +++- 4 files changed, 120 insertions(+), 65 deletions(-) diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index 6c7e6374a4..5e04e6669b 100644 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -449,6 +449,7 @@ func createrLeaderControllers( kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(), ), kubeKyvernoInformer.Core().V1().Secrets(), + kubeKyvernoInformer.Core().V1().ConfigMaps(), kubeInformer.Admissionregistration().V1().MutatingWebhookConfigurations(), kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(), ) diff --git a/pkg/config/config.go b/pkg/config/config.go index 0db2661b1e..e1d28e87f6 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -143,12 +143,17 @@ type configuration struct { } // NewConfiguration ... -func NewConfiguration(client kubernetes.Interface, updateWebhookConfigurations chan<- bool) (Configuration, error) { - cd := &configuration{ +func NewDefaultConfiguration(updateWebhookConfigurations chan<- bool) *configuration { + return &configuration{ updateWebhookConfigurations: updateWebhookConfigurations, restrictDevelopmentUsername: []string{"minikube-user", "kubernetes-admin"}, excludeGroupRole: defaultExcludeGroupRole, } +} + +// NewConfiguration ... +func NewConfiguration(client kubernetes.Interface, updateWebhookConfigurations chan<- bool) (Configuration, error) { + cd := NewDefaultConfiguration(updateWebhookConfigurations) if cm, err := client.CoreV1().ConfigMaps(kyvernoNamespace).Get(context.TODO(), kyvernoConfigMapName, metav1.GetOptions{}); err != nil { if !errors.IsNotFound(err) { return nil, err @@ -227,7 +232,9 @@ func (cd *configuration) Load(cm *corev1.ConfigMap) { } if updateWebhook { logger.Info("webhook configurations changed, updating webhook configurations") - cd.updateWebhookConfigurations <- true + if cd.updateWebhookConfigurations != nil { + cd.updateWebhookConfigurations <- true + } } } diff --git a/pkg/controllers/webhook/controller.go b/pkg/controllers/webhook/controller.go index 943a71d077..cfd6539a44 100644 --- a/pkg/controllers/webhook/controller.go +++ b/pkg/controllers/webhook/controller.go @@ -35,9 +35,10 @@ type controller struct { vwcClient controllerutils.UpdateClient[*admissionregistrationv1.ValidatingWebhookConfiguration] // listers - secretLister corev1listers.SecretLister - mwcLister admissionregistrationv1listers.MutatingWebhookConfigurationLister - vwcLister admissionregistrationv1listers.ValidatingWebhookConfigurationLister + secretLister corev1listers.SecretLister + configMapLister corev1listers.ConfigMapLister + mwcLister admissionregistrationv1listers.MutatingWebhookConfigurationLister + vwcLister admissionregistrationv1listers.ValidatingWebhookConfigurationLister // queue queue workqueue.RateLimitingInterface @@ -50,38 +51,34 @@ func NewController( mwcClient controllerutils.UpdateClient[*admissionregistrationv1.MutatingWebhookConfiguration], vwcClient controllerutils.UpdateClient[*admissionregistrationv1.ValidatingWebhookConfiguration], secretInformer corev1informers.SecretInformer, + configMapInformer corev1informers.ConfigMapInformer, mwcInformer admissionregistrationv1informers.MutatingWebhookConfigurationInformer, vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer, ) controllers.Controller { queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ControllerName) c := controller{ - secretClient: secretClient, - mwcClient: mwcClient, - vwcClient: vwcClient, - secretLister: secretInformer.Lister(), - mwcLister: mwcInformer.Lister(), - vwcLister: vwcInformer.Lister(), - queue: queue, - mwcEnqueue: controllerutils.AddDefaultEventHandlers(logger.V(3), mwcInformer.Informer(), queue), - vwcEnqueue: controllerutils.AddDefaultEventHandlers(logger.V(3), vwcInformer.Informer(), queue), + secretClient: secretClient, + mwcClient: mwcClient, + vwcClient: vwcClient, + secretLister: secretInformer.Lister(), + configMapLister: configMapInformer.Lister(), + mwcLister: mwcInformer.Lister(), + vwcLister: vwcInformer.Lister(), + queue: queue, + mwcEnqueue: controllerutils.AddDefaultEventHandlers(logger.V(3), mwcInformer.Informer(), queue), + vwcEnqueue: controllerutils.AddDefaultEventHandlers(logger.V(3), vwcInformer.Informer(), queue), } - controllerutils.AddEventHandlers( + controllerutils.AddEventHandlersT( secretInformer.Informer(), - func(obj interface{}) { - if err := c.enqueue(obj.(*corev1.Secret)); err != nil { - logger.Error(err, "failed to enqueue") - } - }, - func(_, obj interface{}) { - if err := c.enqueue(obj.(*corev1.Secret)); err != nil { - logger.Error(err, "failed to enqueue") - } - }, - func(obj interface{}) { - if err := c.enqueue(obj.(*corev1.Secret)); err != nil { - logger.Error(err, "failed to enqueue") - } - }, + func(obj *corev1.Secret) { c.secretChanged(obj) }, + func(_, obj *corev1.Secret) { c.secretChanged(obj) }, + func(obj *corev1.Secret) { c.secretChanged(obj) }, + ) + controllerutils.AddEventHandlersT( + configMapInformer.Informer(), + func(obj *corev1.ConfigMap) { c.configMapChanged(obj) }, + func(_, obj *corev1.ConfigMap) { c.configMapChanged(obj) }, + func(obj *corev1.ConfigMap) { c.configMapChanged(obj) }, ) return &c } @@ -90,38 +87,60 @@ func (c *controller) Run(ctx context.Context, workers int) { controllerutils.Run(ctx, ControllerName, logger.V(3), c.queue, workers, maxRetries, c.reconcile) } -func (c *controller) enqueue(obj *corev1.Secret) error { - if obj.GetName() == tls.GenerateRootCASecretName() && obj.GetNamespace() == config.KyvernoNamespace() { - requirement, err := labels.NewRequirement("webhook.kyverno.io/managed-by", selection.Equals, []string{kyvernov1.ValueKyvernoApp}) - if err != nil { - // TODO: log error - return err +func (c *controller) secretChanged(secret *corev1.Secret) { + if secret.GetName() == tls.GenerateRootCASecretName() && secret.GetNamespace() == config.KyvernoNamespace() { + if err := c.enqueueAll(); err != nil { + logger.Error(err, "failed to enqueue on secret change") } - selector := labels.Everything().Add(*requirement) - mwcs, err := c.mwcLister.List(selector) + } +} + +func (c *controller) configMapChanged(cm *corev1.ConfigMap) { + if cm.GetName() == config.KyvernoConfigMapName() && cm.GetNamespace() == config.KyvernoNamespace() { + if err := c.enqueueAll(); err != nil { + logger.Error(err, "failed to enqueue on configmap change") + } + } +} + +func (c *controller) enqueueAll() error { + requirement, err := labels.NewRequirement("webhook.kyverno.io/managed-by", selection.Equals, []string{kyvernov1.ValueKyvernoApp}) + if err != nil { + return err + } + selector := labels.Everything().Add(*requirement) + mwcs, err := c.mwcLister.List(selector) + if err != nil { + return err + } + for _, mwc := range mwcs { + err = c.mwcEnqueue(mwc) if err != nil { return err } - for _, mwc := range mwcs { - err = c.mwcEnqueue(mwc) - if err != nil { - return err - } - } - vwcs, err := c.vwcLister.List(selector) + } + vwcs, err := c.vwcLister.List(selector) + if err != nil { + return err + } + for _, vwc := range vwcs { + err = c.vwcEnqueue(vwc) if err != nil { return err } - for _, vwc := range vwcs { - err = c.vwcEnqueue(vwc) - if err != nil { - return err - } - } } return nil } +func (c *controller) loadConfig() config.Configuration { + cfg := config.NewDefaultConfiguration(nil) + cm, err := c.configMapLister.ConfigMaps(config.KyvernoNamespace()).Get(config.KyvernoConfigMapName()) + if err == nil { + cfg.Load(cm) + } + return cfg +} + func (c *controller) reconcileMutatingWebhookConfiguration(ctx context.Context, logger logr.Logger, name string) error { w, err := c.mwcLister.Get(name) if err != nil { @@ -134,13 +153,21 @@ func (c *controller) reconcileMutatingWebhookConfiguration(ctx context.Context, if labels == nil || labels["webhook.kyverno.io/managed-by"] != kyvernov1.ValueKyvernoApp { return nil } + cfg := c.loadConfig() + webhookCfg := config.WebhookConfig{} + webhookCfgs := cfg.GetWebhooks() + if len(webhookCfgs) > 0 { + webhookCfg = webhookCfgs[0] + } + caData, err := tls.ReadRootCASecret(c.secretClient) + if err != nil { + return err + } _, err = controllerutils.Update(ctx, w, c.mwcClient, func(w *admissionregistrationv1.MutatingWebhookConfiguration) error { - caData, err := tls.ReadRootCASecret(c.secretClient) - if err != nil { - return err - } for i := range w.Webhooks { w.Webhooks[i].ClientConfig.CABundle = caData + w.Webhooks[i].ObjectSelector = webhookCfg.ObjectSelector + w.Webhooks[i].NamespaceSelector = webhookCfg.NamespaceSelector } return nil }) @@ -159,13 +186,21 @@ func (c *controller) reconcileValidatingWebhookConfiguration(ctx context.Context if labels == nil || labels["webhook.kyverno.io/managed-by"] != kyvernov1.ValueKyvernoApp { return nil } + cfg := c.loadConfig() + webhookCfg := config.WebhookConfig{} + webhookCfgs := cfg.GetWebhooks() + if len(webhookCfgs) > 0 { + webhookCfg = webhookCfgs[0] + } + caData, err := tls.ReadRootCASecret(c.secretClient) + if err != nil { + return err + } _, err = controllerutils.Update(ctx, w, c.vwcClient, func(w *admissionregistrationv1.ValidatingWebhookConfiguration) error { - caData, err := tls.ReadRootCASecret(c.secretClient) - if err != nil { - return err - } for i := range w.Webhooks { w.Webhooks[i].ClientConfig.CABundle = caData + w.Webhooks[i].ObjectSelector = webhookCfg.ObjectSelector + w.Webhooks[i].NamespaceSelector = webhookCfg.NamespaceSelector } return nil }) diff --git a/pkg/utils/controller/handlers.go b/pkg/utils/controller/handlers.go index 160b092d66..63b8b83f90 100644 --- a/pkg/utils/controller/handlers.go +++ b/pkg/utils/controller/handlers.go @@ -10,11 +10,15 @@ import ( ) type ( - addFunc func(interface{}) - updateFunc func(interface{}, interface{}) - deleteFunc func(interface{}) - keyFunc func(interface{}) (interface{}, error) - EnqueueFunc func(interface{}) error + addFunc func(interface{}) + updateFunc func(interface{}, interface{}) + deleteFunc func(interface{}) + addFuncT[T any] func(T) + updateFuncT[T any] func(T, T) + deleteFuncT[T any] func(T) + keyFunc func(interface{}) (interface{}, error) + EnqueueFunc func(interface{}) error + EnqueueFuncT[T any] func(T) error ) func AddEventHandlers(informer cache.SharedInformer, a addFunc, u updateFunc, d deleteFunc) { @@ -25,6 +29,14 @@ func AddEventHandlers(informer cache.SharedInformer, a addFunc, u updateFunc, d }) } +func AddEventHandlersT[T any](informer cache.SharedInformer, a addFuncT[T], u updateFuncT[T], d deleteFuncT[T]) { + informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { a(obj.(T)) }, + UpdateFunc: func(old, obj interface{}) { u(old.(T), obj.(T)) }, + DeleteFunc: func(obj interface{}) { d(obj.(T)) }, + }) +} + func AddKeyedEventHandlers(logger logr.Logger, informer cache.SharedInformer, queue workqueue.RateLimitingInterface, parseKey keyFunc) EnqueueFunc { enqueueFunc := LogError(logger, Parse(parseKey, Queue(queue))) AddEventHandlers(informer, AddFunc(logger, enqueueFunc), UpdateFunc(logger, enqueueFunc), DeleteFunc(logger, enqueueFunc))