package webhooks import ( "context" "crypto/tls" "fmt" "net/http" "time" "github.com/go-logr/logr" "github.com/julienschmidt/httprouter" "github.com/kyverno/kyverno/api/kyverno" "github.com/kyverno/kyverno/pkg/clients/dclient" "github.com/kyverno/kyverno/pkg/config" "github.com/kyverno/kyverno/pkg/logging" "github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/toggle" controllerutils "github.com/kyverno/kyverno/pkg/utils/controller" runtimeutils "github.com/kyverno/kyverno/pkg/utils/runtime" "github.com/kyverno/kyverno/pkg/webhooks/handlers" admissionv1 "k8s.io/api/admission/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" rbacv1listers "k8s.io/client-go/listers/rbac/v1" ) // DebugModeOptions holds the options to configure debug mode type DebugModeOptions struct { // DumpPayload is used to activate/deactivate debug mode. DumpPayload bool } type Server interface { // Run TLS server in separate thread and returns control immediately Run() // Stop TLS server and returns control after the server is shut down Stop() } type ExceptionHandlers interface { // Validate performs the validation check on exception resources Validate(context.Context, logr.Logger, handlers.AdmissionRequest, time.Time) admissionv1.AdmissionResponse } type GlobalContextHandlers interface { // Validate performs the validation check on global context entries Validate(context.Context, logr.Logger, handlers.AdmissionRequest, time.Time) admissionv1.AdmissionResponse } type PolicyHandlers interface { // Mutate performs the mutation of policy resources Mutate(context.Context, logr.Logger, handlers.AdmissionRequest, time.Time) admissionv1.AdmissionResponse // Validate performs the validation check on policy resources Validate(context.Context, logr.Logger, handlers.AdmissionRequest, time.Time) admissionv1.AdmissionResponse } type ResourceHandlers interface { // Mutate performs the mutation of kube resources Mutate(context.Context, logr.Logger, handlers.AdmissionRequest, string, time.Time) admissionv1.AdmissionResponse // Validate performs the validation check on kube resources Validate(context.Context, logr.Logger, handlers.AdmissionRequest, string, time.Time) admissionv1.AdmissionResponse } type server struct { server *http.Server runtime runtimeutils.Runtime mwcClient controllerutils.DeleteCollectionClient vwcClient controllerutils.DeleteCollectionClient leaseClient controllerutils.DeleteClient } type TlsProvider func() ([]byte, []byte, error) // NewServer creates new instance of server accordingly to given configuration func NewServer( ctx context.Context, policyHandlers PolicyHandlers, resourceHandlers ResourceHandlers, exceptionHandlers ExceptionHandlers, globalContextHandlers GlobalContextHandlers, configuration config.Configuration, metricsConfig metrics.MetricsConfigManager, debugModeOpts DebugModeOptions, tlsProvider TlsProvider, mwcClient controllerutils.DeleteCollectionClient, vwcClient controllerutils.DeleteCollectionClient, leaseClient controllerutils.DeleteClient, runtime runtimeutils.Runtime, rbLister rbacv1listers.RoleBindingLister, crbLister rbacv1listers.ClusterRoleBindingLister, discovery dclient.IDiscovery, webhookServerPort int32, ) Server { mux := httprouter.New() resourceLogger := logger.WithName("resource") policyLogger := logger.WithName("policy") exceptionLogger := logger.WithName("exception") globalContextLogger := logger.WithName("globalcontext") verifyLogger := logger.WithName("verify") registerWebhookHandlers( mux, "MUTATE", config.MutatingWebhookServicePath, resourceHandlers.Mutate, func(handler handlers.AdmissionHandler) handlers.HttpHandler { return handler. WithFilter(configuration). WithProtection(toggle.FromContext(ctx).ProtectManagedResources()). WithDump(debugModeOpts.DumpPayload). WithTopLevelGVK(discovery). WithRoles(rbLister, crbLister). WithOperationFilter(admissionv1.Create, admissionv1.Update, admissionv1.Connect). WithMetrics(resourceLogger, metricsConfig.Config(), metrics.WebhookMutating). WithAdmission(resourceLogger.WithName("mutate")) }, ) registerWebhookHandlers( mux, "VALIDATE", config.ValidatingWebhookServicePath, resourceHandlers.Validate, func(handler handlers.AdmissionHandler) handlers.HttpHandler { return handler. WithFilter(configuration). WithProtection(toggle.FromContext(ctx).ProtectManagedResources()). WithDump(debugModeOpts.DumpPayload). WithTopLevelGVK(discovery). WithRoles(rbLister, crbLister). WithMetrics(resourceLogger, metricsConfig.Config(), metrics.WebhookValidating). WithAdmission(resourceLogger.WithName("validate")) }, ) mux.HandlerFunc( "POST", config.PolicyMutatingWebhookServicePath, handlers.FromAdmissionFunc("MUTATE", policyHandlers.Mutate). WithDump(debugModeOpts.DumpPayload). WithMetrics(policyLogger, metricsConfig.Config(), metrics.WebhookMutating). WithAdmission(policyLogger.WithName("mutate")). ToHandlerFunc("MUTATE"), ) mux.HandlerFunc( "POST", config.PolicyValidatingWebhookServicePath, handlers.FromAdmissionFunc("VALIDATE", policyHandlers.Validate). WithDump(debugModeOpts.DumpPayload). WithSubResourceFilter(). WithMetrics(policyLogger, metricsConfig.Config(), metrics.WebhookValidating). WithAdmission(policyLogger.WithName("validate")). ToHandlerFunc("VALIDATE"), ) mux.HandlerFunc( "POST", config.ExceptionValidatingWebhookServicePath, handlers.FromAdmissionFunc("VALIDATE", exceptionHandlers.Validate). WithDump(debugModeOpts.DumpPayload). WithSubResourceFilter(). WithMetrics(exceptionLogger, metricsConfig.Config(), metrics.WebhookValidating). WithAdmission(exceptionLogger.WithName("validate")). ToHandlerFunc("VALIDATE"), ) mux.HandlerFunc( "POST", config.GlobalContextValidatingWebhookServicePath, handlers.FromAdmissionFunc("VALIDATE", globalContextHandlers.Validate). WithDump(debugModeOpts.DumpPayload). WithSubResourceFilter(). WithMetrics(globalContextLogger, metricsConfig.Config(), metrics.WebhookValidating). WithAdmission(globalContextLogger.WithName("validate")). ToHandlerFunc("VALIDATE"), ) mux.HandlerFunc( "POST", config.VerifyMutatingWebhookServicePath, handlers.FromAdmissionFunc("VERIFY", handlers.Verify). WithAdmission(verifyLogger.WithName("mutate")). ToHandlerFunc("VERIFY"), ) mux.HandlerFunc("GET", config.LivenessServicePath, handlers.Probe(runtime.IsLive)) mux.HandlerFunc("GET", config.ReadinessServicePath, handlers.Probe(runtime.IsReady)) return &server{ server: &http.Server{ Addr: fmt.Sprintf(":%d", webhookServerPort), TLSConfig: &tls.Config{ GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { certPem, keyPem, err := tlsProvider() if err != nil { return nil, err } pair, err := tls.X509KeyPair(certPem, keyPem) if err != nil { return nil, err } return &pair, nil }, MinVersion: tls.VersionTLS12, CipherSuites: []uint16{ // AEADs w/ ECDHE tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, }, }, Handler: mux, ReadTimeout: 30 * time.Second, WriteTimeout: 30 * time.Second, ReadHeaderTimeout: 30 * time.Second, IdleTimeout: 5 * time.Minute, ErrorLog: logging.StdLogger(logger.WithName("server"), ""), }, mwcClient: mwcClient, vwcClient: vwcClient, leaseClient: leaseClient, runtime: runtime, } } func (s *server) Run() { go func() { if err := s.server.ListenAndServeTLS("", ""); err != nil { logging.Error(err, "failed to start server") } }() } func (s *server) Stop() { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() s.cleanup(ctx) err := s.server.Shutdown(ctx) if err != nil { logger.Error(err, "shutting down server") err = s.server.Close() if err != nil { logger.Error(err, "server shut down failed") } } } func (s *server) cleanup(ctx context.Context) { if s.runtime.IsGoingDown() { deleteLease := func(name string) { if err := s.leaseClient.Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { logger.Error(err, "failed to clean up lease", "name", name) } else if err == nil { logger.Info("successfully deleted leases", "label", kyverno.LabelWebhookManagedBy) } } deleteVwc := func() { if err := s.vwcClient.DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ LabelSelector: kyverno.LabelWebhookManagedBy, }); err != nil && !apierrors.IsNotFound(err) { logger.Error(err, "failed to clean up validating webhook configuration", "label", kyverno.LabelWebhookManagedBy) } else if err == nil { logger.Info("successfully deleted validating webhook configurations", "label", kyverno.LabelWebhookManagedBy) } } deleteMwc := func() { if err := s.mwcClient.DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ LabelSelector: kyverno.LabelWebhookManagedBy, }); err != nil && !apierrors.IsNotFound(err) { logger.Error(err, "failed to clean up mutating webhook configuration", "label", kyverno.LabelWebhookManagedBy) } else if err == nil { logger.Info("successfully deleted mutating webhook configurations", "label", kyverno.LabelWebhookManagedBy) } } deleteLease("kyvernopre-lock") deleteLease("kyverno-health") deleteVwc() deleteMwc() } } func registerWebhookHandlers( mux *httprouter.Router, name string, basePath string, handlerFunc func(context.Context, logr.Logger, handlers.AdmissionRequest, string, time.Time) admissionv1.AdmissionResponse, builder func(handler handlers.AdmissionHandler) handlers.HttpHandler, ) { all := handlers.FromAdmissionFunc( name, func(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, startTime time.Time) admissionv1.AdmissionResponse { return handlerFunc(ctx, logger, request, "all", startTime) }, ) ignore := handlers.FromAdmissionFunc( name, func(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, startTime time.Time) admissionv1.AdmissionResponse { return handlerFunc(ctx, logger, request, "ignore", startTime) }, ) fail := handlers.FromAdmissionFunc( name, func(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, startTime time.Time) admissionv1.AdmissionResponse { return handlerFunc(ctx, logger, request, "fail", startTime) }, ) mux.HandlerFunc("POST", basePath, builder(all).ToHandlerFunc(name)) mux.HandlerFunc("POST", basePath+"/ignore", builder(ignore).ToHandlerFunc(name)) mux.HandlerFunc("POST", basePath+"/fail", builder(fail).ToHandlerFunc(name)) mux.HandlerFunc("POST", basePath+"/ignore"+config.FineGrainedWebhookPath+"/*policy", builder(ignore).ToHandlerFunc(name)) mux.HandlerFunc("POST", basePath+"/fail"+config.FineGrainedWebhookPath+"/*policy", builder(fail).ToHandlerFunc(name)) }