package webhook import ( "context" "fmt" "time" "github.com/go-logr/logr" "github.com/kyverno/kyverno/api/kyverno" "github.com/kyverno/kyverno/pkg/config" "github.com/kyverno/kyverno/pkg/controllers" "github.com/kyverno/kyverno/pkg/logging" "github.com/kyverno/kyverno/pkg/tls" controllerutils "github.com/kyverno/kyverno/pkg/utils/controller" runtimeutils "github.com/kyverno/kyverno/pkg/utils/runtime" "golang.org/x/exp/maps" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" admissionregistrationv1informers "k8s.io/client-go/informers/admissionregistration/v1" appsv1informers "k8s.io/client-go/informers/apps/v1" corev1informers "k8s.io/client-go/informers/core/v1" admissionregistrationv1listers "k8s.io/client-go/listers/admissionregistration/v1" appsv1listers "k8s.io/client-go/listers/apps/v1" corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/util/workqueue" ) const ( maxRetries = 10 ) var ( none = admissionregistrationv1.SideEffectClassNone fail = admissionregistrationv1.Fail ignore = admissionregistrationv1.Ignore None = &none Fail = &fail Ignore = &ignore ) type controller struct { // clients vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration] // listers vwcLister admissionregistrationv1listers.ValidatingWebhookConfigurationLister secretLister corev1listers.SecretNamespaceLister deploymentLister appsv1listers.DeploymentNamespaceLister // queue queue workqueue.TypedRateLimitingInterface[any] // config controllerName string logger logr.Logger webhookName string path string server string servicePort int32 rules []admissionregistrationv1.RuleWithOperations failurePolicy *admissionregistrationv1.FailurePolicyType sideEffects *admissionregistrationv1.SideEffectClass runtime runtimeutils.Runtime configuration config.Configuration labelSelector *metav1.LabelSelector caSecretName string webhooksDeleted bool autoDeleteWebhooks bool webhookCleanupSetup func(context.Context, logr.Logger) error postWebhookCleanup func(context.Context, logr.Logger) error } func NewController( controllerName string, vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration], vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer, secretInformer corev1informers.SecretInformer, deploymentInformer appsv1informers.DeploymentInformer, webhookName string, path string, server string, servicePort int32, webhookServerPort int32, labelSelector *metav1.LabelSelector, rules []admissionregistrationv1.RuleWithOperations, failurePolicy *admissionregistrationv1.FailurePolicyType, sideEffects *admissionregistrationv1.SideEffectClass, configuration config.Configuration, caSecretName string, runtime runtimeutils.Runtime, autoDeleteWebhooks bool, webhookCleanupSetup func(context.Context, logr.Logger) error, postWebhookCleanup func(context.Context, logr.Logger) error, ) controllers.Controller { queue := workqueue.NewTypedRateLimitingQueueWithConfig( workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: controllerName}, ) c := controller{ vwcClient: vwcClient, vwcLister: vwcInformer.Lister(), secretLister: secretInformer.Lister().Secrets(config.KyvernoNamespace()), deploymentLister: deploymentInformer.Lister().Deployments(config.KyvernoNamespace()), queue: queue, controllerName: controllerName, logger: logging.ControllerLogger(controllerName), webhookName: webhookName, path: path, server: server, servicePort: servicePort, rules: rules, failurePolicy: failurePolicy, sideEffects: sideEffects, configuration: configuration, labelSelector: labelSelector, caSecretName: caSecretName, runtime: runtime, autoDeleteWebhooks: autoDeleteWebhooks, webhookCleanupSetup: webhookCleanupSetup, postWebhookCleanup: postWebhookCleanup, } if _, _, err := controllerutils.AddDefaultEventHandlers(c.logger, vwcInformer.Informer(), queue); err != nil { c.logger.Error(err, "failed to register event handlers") } if _, err := controllerutils.AddEventHandlersT( secretInformer.Informer(), func(obj *corev1.Secret) { if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == caSecretName { c.enqueue() } }, func(_, obj *corev1.Secret) { if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == caSecretName { c.enqueue() } }, func(obj *corev1.Secret) { if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == caSecretName { c.enqueue() } }, ); err != nil { c.logger.Error(err, "failed to register event handlers") } if autoDeleteWebhooks { if _, err := controllerutils.AddEventHandlersT( deploymentInformer.Informer(), func(obj *appsv1.Deployment) { }, func(_, obj *appsv1.Deployment) { if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoDeploymentName() { c.enqueueCleanupAfter(1 * time.Second) } }, func(obj *appsv1.Deployment) { if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoDeploymentName() { c.enqueueCleanup() } }, ); err != nil { c.logger.Error(err, "failed to register event handlers") } } configuration.OnChanged(c.enqueue) return &c } func (c *controller) Run(ctx context.Context, workers int) { if c.autoDeleteWebhooks { if err := c.webhookCleanupSetup(ctx, c.logger); err != nil { c.logger.Error(err, "failed to setup webhook cleanup") } } c.enqueue() controllerutils.Run(ctx, c.logger, c.controllerName, time.Second, c.queue, workers, maxRetries, c.reconcile) } func (c *controller) enqueue() { c.queue.Add(c.webhookName) } func (c *controller) enqueueCleanup() { c.queue.Add(config.KyvernoDeploymentName()) } func (c *controller) enqueueCleanupAfter(duration time.Duration) { c.queue.AddAfter(config.KyvernoDeploymentName(), duration) } func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, _, _ string) error { if c.autoDeleteWebhooks && c.runtime.IsGoingDown() { return c.reconcileWebhookDeletion(ctx) } if c.autoDeleteWebhooks && key == config.KyvernoDeploymentName() { return c.reconcileWebhookDeletion(ctx) } if key != c.webhookName { return nil } caData, err := tls.ReadRootCASecret(c.caSecretName, config.KyvernoNamespace(), c.secretLister) if err != nil { return err } desired, err := c.build(c.configuration, caData) if err != nil { return err } observed, err := c.vwcLister.Get(desired.Name) if err != nil { if apierrors.IsNotFound(err) { _, err := c.vwcClient.Create(ctx, desired, metav1.CreateOptions{}) return err } return err } _, err = controllerutils.Update(ctx, observed, c.vwcClient, func(w *admissionregistrationv1.ValidatingWebhookConfiguration) error { w.Labels = desired.Labels w.Annotations = desired.Annotations w.OwnerReferences = desired.OwnerReferences w.Webhooks = desired.Webhooks return nil }) return err } func (c *controller) reconcileWebhookDeletion(ctx context.Context) error { if c.autoDeleteWebhooks { if c.runtime.IsGoingDown() { if c.webhooksDeleted { return nil } c.webhooksDeleted = true if err := c.vwcClient.Delete(ctx, c.webhookName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { c.logger.Error(err, "failed to clean up validating webhook configuration", "label", kyverno.LabelWebhookManagedBy) return err } else if err == nil { c.logger.V(4).Info("successfully deleted validating webhook configurations", "label", kyverno.LabelWebhookManagedBy) } if err := c.postWebhookCleanup(ctx, c.logger); err != nil { c.logger.Error(err, "failed to clean up temporary rbac") return err } else { c.logger.V(4).Info("successfully deleted temporary rbac") } } else { if err := c.webhookCleanupSetup(ctx, c.logger); err != nil { c.logger.Error(err, "failed to reconcile webhook cleanup setup") return err } c.logger.V(4).Info("reconciled webhook cleanup setup") } } return nil } func objectMeta(name string, annotations map[string]string, labels map[string]string, owner ...metav1.OwnerReference) metav1.ObjectMeta { desiredLabels := make(map[string]string) defaultLabels := map[string]string{ kyverno.LabelWebhookManagedBy: kyverno.ValueKyvernoApp, } maps.Copy(desiredLabels, labels) maps.Copy(desiredLabels, defaultLabels) return metav1.ObjectMeta{ Name: name, Labels: desiredLabels, Annotations: annotations, OwnerReferences: owner, } } func (c *controller) build(cfg config.Configuration, caBundle []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) { return &admissionregistrationv1.ValidatingWebhookConfiguration{ ObjectMeta: objectMeta(c.webhookName, cfg.GetWebhookAnnotations(), cfg.GetWebhookLabels()), Webhooks: []admissionregistrationv1.ValidatingWebhook{{ Name: fmt.Sprintf("%s.%s.svc", config.KyvernoServiceName(), config.KyvernoNamespace()), ClientConfig: c.clientConfig(caBundle), Rules: c.rules, FailurePolicy: c.failurePolicy, SideEffects: c.sideEffects, AdmissionReviewVersions: []string{"v1"}, ObjectSelector: c.labelSelector, MatchConditions: cfg.GetMatchConditions(), }}, }, nil } func (c *controller) clientConfig(caBundle []byte) admissionregistrationv1.WebhookClientConfig { clientConfig := admissionregistrationv1.WebhookClientConfig{ CABundle: caBundle, } if c.server == "" { clientConfig.Service = &admissionregistrationv1.ServiceReference{ Namespace: config.KyvernoNamespace(), Name: config.KyvernoServiceName(), Path: &c.path, Port: &c.servicePort, } } else { url := fmt.Sprintf("https://%s%s", c.server, c.path) clientConfig.URL = &url } return clientConfig }