From 4aed9359cb159dc804a6b031c0829328d5a561eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?=
<charled.breteche@gmail.com>
Date: Wed, 12 Oct 2022 08:52:42 +0200
Subject: [PATCH] refactor: manage webhooks with webhook controller (#4846)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* refactor: add config support to webhook controller
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* refactor: add client config to webhook controller
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* migrate verify webhook
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* fix
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* v1
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* refactor: move policy webhooks management in webhook controller
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* policy validating webhook config
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* watch policies
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* refactor: migrate resource webhook management in webhook controller
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* mutating webhook
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* auto update
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* cleanup
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* auto update and wildcard policies
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* policy readiness
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* fix: can't use v1 admission
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* reduce reconcile
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* watchdog
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* cleanup
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* fix
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* health check
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* runtime utils
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* runtime utils
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* cleanup
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* watchdog check
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* remove delete from mutating webhook
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
* cleanup
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Co-authored-by: shuting <shuting@nirmata.com>
---
api/kyverno/v1/clusterpolicy_types.go | 5 +
api/kyverno/v1/policy_interface.go | 1 +
api/kyverno/v1/policy_types.go | 5 +
cmd/kyverno/main.go | 96 ++-
pkg/config/config.go | 21 +-
pkg/controllers/webhook/controller.go | 875 ++++++++++++++++++++----
pkg/controllers/webhook/log.go | 2 +-
pkg/controllers/webhook/utils.go | 72 ++
pkg/controllers/webhook/utils_test.go | 16 +
pkg/utils/runtime/utils.go | 121 ++++
pkg/webhookconfig/common.go | 383 -----------
pkg/webhookconfig/configmanager.go | 681 ------------------
pkg/webhookconfig/configmanager_test.go | 16 -
pkg/webhookconfig/monitor.go | 235 -------
pkg/webhookconfig/registration.go | 653 ------------------
pkg/webhookconfig/status.go | 134 ----
pkg/webhooks/handlers/admission.go | 4 +-
pkg/webhooks/handlers/monitor.go | 15 -
pkg/webhooks/handlers/probe.go | 4 +-
pkg/webhooks/server.go | 87 ++-
20 files changed, 1087 insertions(+), 2339 deletions(-)
create mode 100644 pkg/controllers/webhook/utils.go
create mode 100644 pkg/controllers/webhook/utils_test.go
create mode 100644 pkg/utils/runtime/utils.go
delete mode 100644 pkg/webhookconfig/common.go
delete mode 100644 pkg/webhookconfig/configmanager.go
delete mode 100644 pkg/webhookconfig/configmanager_test.go
delete mode 100644 pkg/webhookconfig/monitor.go
delete mode 100644 pkg/webhookconfig/registration.go
delete mode 100644 pkg/webhookconfig/status.go
delete mode 100644 pkg/webhooks/handlers/monitor.go
diff --git a/api/kyverno/v1/clusterpolicy_types.go b/api/kyverno/v1/clusterpolicy_types.go
index 78baa2f619..540cfe048f 100644
--- a/api/kyverno/v1/clusterpolicy_types.go
+++ b/api/kyverno/v1/clusterpolicy_types.go
@@ -83,6 +83,11 @@ func (p *ClusterPolicy) GetSpec() *Spec {
return &p.Spec
}
+// GetStatus returns the policy status
+func (p *ClusterPolicy) GetStatus() *PolicyStatus {
+ return &p.Status
+}
+
// IsNamespaced indicates if the policy is namespace scoped
func (p *ClusterPolicy) IsNamespaced() bool {
return p.GetNamespace() != ""
diff --git a/api/kyverno/v1/policy_interface.go b/api/kyverno/v1/policy_interface.go
index 752359ff24..e5c2eb8f85 100644
--- a/api/kyverno/v1/policy_interface.go
+++ b/api/kyverno/v1/policy_interface.go
@@ -14,6 +14,7 @@ type PolicyInterface interface {
HasAutoGenAnnotation() bool
IsNamespaced() bool
GetSpec() *Spec
+ GetStatus() *PolicyStatus
Validate(sets.String) field.ErrorList
GetKind() string
CreateDeepCopy() PolicyInterface
diff --git a/api/kyverno/v1/policy_types.go b/api/kyverno/v1/policy_types.go
index 9aef36fce3..57a55ee868 100755
--- a/api/kyverno/v1/policy_types.go
+++ b/api/kyverno/v1/policy_types.go
@@ -84,6 +84,11 @@ func (p *Policy) GetSpec() *Spec {
return &p.Spec
}
+// GetStatus returns the policy status
+func (p *Policy) GetStatus() *PolicyStatus {
+ return &p.Status
+}
+
// IsNamespaced indicates if the policy is namespace scoped
func (p *Policy) IsNamespaced() bool {
return true
diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go
index 014a5dac46..4970707a06 100644
--- a/cmd/kyverno/main.go
+++ b/cmd/kyverno/main.go
@@ -20,7 +20,6 @@ import (
kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions"
"github.com/kyverno/kyverno/pkg/clients/dclient"
kyvernoclient "github.com/kyverno/kyverno/pkg/clients/wrappers"
- "github.com/kyverno/kyverno/pkg/common"
"github.com/kyverno/kyverno/pkg/config"
"github.com/kyverno/kyverno/pkg/controllers/certmanager"
configcontroller "github.com/kyverno/kyverno/pkg/controllers/config"
@@ -44,14 +43,15 @@ import (
"github.com/kyverno/kyverno/pkg/toggle"
"github.com/kyverno/kyverno/pkg/tracing"
"github.com/kyverno/kyverno/pkg/utils"
+ runtimeutils "github.com/kyverno/kyverno/pkg/utils/runtime"
"github.com/kyverno/kyverno/pkg/version"
- "github.com/kyverno/kyverno/pkg/webhookconfig"
"github.com/kyverno/kyverno/pkg/webhooks"
webhookspolicy "github.com/kyverno/kyverno/pkg/webhooks/policy"
webhooksresource "github.com/kyverno/kyverno/pkg/webhooks/resource"
webhookgenerate "github.com/kyverno/kyverno/pkg/webhooks/updaterequest"
_ "go.uber.org/automaxprocs" // #nosec
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+ coordinationv1 "k8s.io/api/coordination/v1"
corev1 "k8s.io/api/core/v1"
kubeinformers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
@@ -99,7 +99,7 @@ var (
func parseFlags() error {
logging.Init(nil)
flag.StringVar(&logFormat, "loggingFormat", logging.TextFormat, "This determines the output format of the logger.")
- flag.IntVar(&webhookTimeout, "webhookTimeout", int(webhookconfig.DefaultWebhookTimeout), "Timeout for webhook configurations.")
+ flag.IntVar(&webhookTimeout, "webhookTimeout", webhookcontroller.DefaultWebhookTimeout, "Timeout for webhook configurations.")
flag.IntVar(&genWorkers, "genWorkers", 10, "Workers for generate controller.")
flag.IntVar(&maxQueuedEvents, "maxQueuedEvents", 1000, "Maximum events to be queued.")
flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
@@ -414,6 +414,7 @@ func createrLeaderControllers(
metricsConfig *metrics.MetricsConfig,
eventGenerator event.Interface,
certRenewer *tls.CertRenewer,
+ runtime runtimeutils.Runtime,
) ([]controller, error) {
policyCtrl, err := policy.NewPolicyController(
kyvernoClient,
@@ -436,6 +437,7 @@ func createrLeaderControllers(
certRenewer,
)
webhookController := webhookcontroller.NewController(
+ dynamicClient.Discovery(),
metrics.ObjectClient[*corev1.Secret](
metrics.NamespacedClientQueryRecorder(metricsConfig, config.KyvernoNamespace(), "Secret", metrics.KubeClient),
kubeClient.CoreV1().Secrets(config.KyvernoNamespace()),
@@ -448,10 +450,22 @@ func createrLeaderControllers(
metrics.ClusteredClientQueryRecorder(metricsConfig, "ValidatingWebhookConfiguration", metrics.KubeClient),
kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(),
),
- kubeKyvernoInformer.Core().V1().Secrets(),
- kubeKyvernoInformer.Core().V1().ConfigMaps(),
+ metrics.ObjectClient[*coordinationv1.Lease](
+ metrics.ClusteredClientQueryRecorder(metricsConfig, "Lease", metrics.KubeClient),
+ kubeClient.CoordinationV1().Leases(config.KyvernoNamespace()),
+ ),
+ kyvernoClient,
kubeInformer.Admissionregistration().V1().MutatingWebhookConfigurations(),
kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(),
+ kyvernoInformer.Kyverno().V1().ClusterPolicies(),
+ kyvernoInformer.Kyverno().V1().Policies(),
+ kubeKyvernoInformer.Core().V1().Secrets(),
+ kubeKyvernoInformer.Core().V1().ConfigMaps(),
+ kubeKyvernoInformer.Coordination().V1().Leases(),
+ serverIP,
+ int32(webhookTimeout),
+ autoUpdateWebhooks,
+ runtime,
)
return append(
[]controller{
@@ -537,26 +551,8 @@ func main() {
kubeInformer := kubeinformers.NewSharedInformerFactory(kubeClient, resyncPeriod)
kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace()))
kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(kyvernoClient, resyncPeriod)
- webhookCfg := webhookconfig.NewRegister(
- signalCtx,
- clientConfig,
- dynamicClient,
- kubeClient,
- kyvernoClient,
- kubeInformer.Admissionregistration().V1().MutatingWebhookConfigurations(),
- kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(),
- kubeKyvernoInformer.Apps().V1().Deployments(),
- kyvernoInformer.Kyverno().V1().ClusterPolicies(),
- kyvernoInformer.Kyverno().V1().Policies(),
- metricsConfig,
- serverIP,
- int32(webhookTimeout),
- autoUpdateWebhooks,
- logging.GlobalLogger(),
- )
configuration, err := config.NewConfiguration(
kubeClient,
- webhookCfg.UpdateWebhookChan,
)
if err != nil {
logger.Error(err, "failed to initialize configuration")
@@ -591,12 +587,18 @@ func main() {
maxQueuedEvents,
logging.WithName("EventGenerator"),
)
- // This controller only subscribe to events, nothing is returned...
+ // this controller only subscribe to events, nothing is returned...
policymetricscontroller.NewController(
metricsConfig,
kyvernoInformer.Kyverno().V1().ClusterPolicies(),
kyvernoInformer.Kyverno().V1().Policies(),
)
+ runtime := runtimeutils.NewRuntime(
+ logger.WithName("runtime"),
+ serverIP,
+ kubeKyvernoInformer.Coordination().V1().Leases(),
+ kubeKyvernoInformer.Apps().V1().Deployments(),
+ )
// create non leader controllers
nonLeaderControllers, nonLeaderBootstrap := createNonLeaderControllers(
kubeInformer,
@@ -640,10 +642,10 @@ func main() {
// when losing the lead we just terminate the pod
defer signalCancel()
// validate config
- if err := webhookCfg.ValidateWebhookConfigurations(config.KyvernoNamespace(), config.KyvernoConfigMapName()); err != nil {
- logger.Error(err, "invalid format of the Kyverno init ConfigMap, please correct the format of 'data.webhooks'")
- os.Exit(1)
- }
+ // if err := webhookCfg.ValidateWebhookConfigurations(config.KyvernoNamespace(), config.KyvernoConfigMapName()); err != nil {
+ // logger.Error(err, "invalid format of the Kyverno init ConfigMap, please correct the format of 'data.webhooks'")
+ // os.Exit(1)
+ // }
// create leader factories
kubeInformer := kubeinformers.NewSharedInformerFactory(kubeClient, resyncPeriod)
kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace()))
@@ -662,6 +664,7 @@ func main() {
metricsConfig,
eventGenerator,
certRenewer,
+ runtime,
)
if err != nil {
logger.Error(err, "failed to create leader controllers")
@@ -681,16 +684,6 @@ func main() {
for _, controller := range leaderControllers {
go controller.run(signalCtx, logger.WithName("controllers"))
}
- // bootstrap
- if autoUpdateWebhooks {
- go webhookCfg.UpdateWebhookConfigurations(configuration)
- }
- registerWrapperRetry := common.RetryFunc(time.Second, webhookRegistrationTimeout, webhookCfg.Register, "failed to register webhook", logger)
- if err := registerWrapperRetry(); err != nil {
- logger.Error(err, "timeout registering admission control webhooks")
- os.Exit(1)
- }
- webhookCfg.UpdateWebhookChan <- true
// wait until we loose the lead (or signal context is canceled)
<-ctx.Done()
},
@@ -702,16 +695,6 @@ func main() {
}
// start leader election
go le.Run(signalCtx)
- // create monitor
- webhookMonitor, err := webhookconfig.NewMonitor(kubeClient, logging.GlobalLogger())
- if err != nil {
- logger.Error(err, "failed to initialize webhookMonitor")
- os.Exit(1)
- }
- // start monitor (only when running in cluster)
- if serverIP == "" {
- go webhookMonitor.Run(signalCtx, webhookCfg, certRenewer, eventGenerator)
- }
// create webhooks server
urgen := webhookgenerate.NewGenerator(
kyvernoClient,
@@ -740,6 +723,7 @@ func main() {
server := webhooks.NewServer(
policyHandlers,
resourceHandlers,
+ configuration,
func() ([]byte, []byte, error) {
secret, err := secretLister.Secrets(config.KyvernoNamespace()).Get(tls.GenerateTLSPairSecretName())
if err != nil {
@@ -747,9 +731,19 @@ func main() {
}
return secret.Data[corev1.TLSCertKey], secret.Data[corev1.TLSPrivateKeyKey], nil
},
- configuration,
- webhookCfg,
- webhookMonitor,
+ metrics.ObjectClient[*admissionregistrationv1.MutatingWebhookConfiguration](
+ metrics.ClusteredClientQueryRecorder(metricsConfig, "MutatingWebhookConfiguration", metrics.KubeClient),
+ kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations(),
+ ),
+ metrics.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration](
+ metrics.ClusteredClientQueryRecorder(metricsConfig, "ValidatingWebhookConfiguration", metrics.KubeClient),
+ kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(),
+ ),
+ metrics.ObjectClient[*coordinationv1.Lease](
+ metrics.ClusteredClientQueryRecorder(metricsConfig, "Lease", metrics.KubeClient),
+ kubeClient.CoordinationV1().Leases(config.KyvernoNamespace()),
+ ),
+ runtime,
)
// start informers and wait for cache sync
// we need to call start again because we potentially registered new informers
diff --git a/pkg/config/config.go b/pkg/config/config.go
index e1d28e87f6..56f1481dba 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -18,32 +18,22 @@ import (
const (
// MutatingWebhookConfigurationName default resource mutating webhook configuration name
MutatingWebhookConfigurationName = "kyverno-resource-mutating-webhook-cfg"
- // MutatingWebhookConfigurationDebugName default resource mutating webhook configuration name for debug mode
- MutatingWebhookConfigurationDebugName = "kyverno-resource-mutating-webhook-cfg-debug"
// MutatingWebhookName default resource mutating webhook name
MutatingWebhookName = "mutate.kyverno.svc"
// ValidatingWebhookConfigurationName ...
ValidatingWebhookConfigurationName = "kyverno-resource-validating-webhook-cfg"
- // ValidatingWebhookConfigurationDebugName ...
- ValidatingWebhookConfigurationDebugName = "kyverno-resource-validating-webhook-cfg-debug"
// ValidatingWebhookName ...
ValidatingWebhookName = "validate.kyverno.svc"
// VerifyMutatingWebhookConfigurationName default verify mutating webhook configuration name
VerifyMutatingWebhookConfigurationName = "kyverno-verify-mutating-webhook-cfg"
- // VerifyMutatingWebhookConfigurationDebugName default verify mutating webhook configuration name for debug mode
- VerifyMutatingWebhookConfigurationDebugName = "kyverno-verify-mutating-webhook-cfg-debug"
// VerifyMutatingWebhookName default verify mutating webhook name
VerifyMutatingWebhookName = "monitor-webhooks.kyverno.svc"
// PolicyValidatingWebhookConfigurationName default policy validating webhook configuration name
PolicyValidatingWebhookConfigurationName = "kyverno-policy-validating-webhook-cfg"
- // PolicyValidatingWebhookConfigurationDebugName default policy validating webhook configuration name for debug mode
- PolicyValidatingWebhookConfigurationDebugName = "kyverno-policy-validating-webhook-cfg-debug"
// PolicyValidatingWebhookName default policy validating webhook name
PolicyValidatingWebhookName = "validate-policy.kyverno.svc"
// PolicyMutatingWebhookConfigurationName default policy mutating webhook configuration name
PolicyMutatingWebhookConfigurationName = "kyverno-policy-mutating-webhook-cfg"
- // PolicyMutatingWebhookConfigurationDebugName default policy mutating webhook configuration name for debug mode
- PolicyMutatingWebhookConfigurationDebugName = "kyverno-policy-mutating-webhook-cfg-debug"
// PolicyMutatingWebhookName default policy mutating webhook name
PolicyMutatingWebhookName = "mutate-policy.kyverno.svc"
// Due to kubernetes issue, we must use next literal constants instead of deployment TypeMeta fields
@@ -139,21 +129,19 @@ type configuration struct {
restrictDevelopmentUsername []string
webhooks []WebhookConfig
generateSuccessEvents bool
- updateWebhookConfigurations chan<- bool
}
// NewConfiguration ...
-func NewDefaultConfiguration(updateWebhookConfigurations chan<- bool) *configuration {
+func NewDefaultConfiguration() *configuration {
return &configuration{
- updateWebhookConfigurations: updateWebhookConfigurations,
restrictDevelopmentUsername: []string{"minikube-user", "kubernetes-admin"},
excludeGroupRole: defaultExcludeGroupRole,
}
}
// NewConfiguration ...
-func NewConfiguration(client kubernetes.Interface, updateWebhookConfigurations chan<- bool) (Configuration, error) {
- cd := NewDefaultConfiguration(updateWebhookConfigurations)
+func NewConfiguration(client kubernetes.Interface) (Configuration, error) {
+ cd := NewDefaultConfiguration()
if cm, err := client.CoreV1().ConfigMaps(kyvernoNamespace).Get(context.TODO(), kyvernoConfigMapName, metav1.GetOptions{}); err != nil {
if !errors.IsNotFound(err) {
return nil, err
@@ -232,9 +220,6 @@ func (cd *configuration) Load(cm *corev1.ConfigMap) {
}
if updateWebhook {
logger.Info("webhook configurations changed, updating webhook configurations")
- if cd.updateWebhookConfigurations != nil {
- cd.updateWebhookConfigurations <- true
- }
}
}
diff --git a/pkg/controllers/webhook/controller.go b/pkg/controllers/webhook/controller.go
index cfd6539a44..841b7745de 100644
--- a/pkg/controllers/webhook/controller.go
+++ b/pkg/controllers/webhook/controller.go
@@ -1,139 +1,283 @@
-package background
+package webhook
import (
"context"
+ "fmt"
+ "strings"
+ "sync"
+ "time"
"github.com/go-logr/logr"
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
+ "github.com/kyverno/kyverno/pkg/autogen"
+ "github.com/kyverno/kyverno/pkg/client/clientset/versioned"
+ kyvernov1informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1"
+ kyvernov1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1"
+ "github.com/kyverno/kyverno/pkg/clients/dclient"
"github.com/kyverno/kyverno/pkg/config"
"github.com/kyverno/kyverno/pkg/controllers"
"github.com/kyverno/kyverno/pkg/tls"
controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
+ kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
+ runtimeutils "github.com/kyverno/kyverno/pkg/utils/runtime"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+ coordinationv1 "k8s.io/api/coordination/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
- "k8s.io/apimachinery/pkg/selection"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/apimachinery/pkg/util/sets"
admissionregistrationv1informers "k8s.io/client-go/informers/admissionregistration/v1"
+ coordinationv1informers "k8s.io/client-go/informers/coordination/v1"
corev1informers "k8s.io/client-go/informers/core/v1"
admissionregistrationv1listers "k8s.io/client-go/listers/admissionregistration/v1"
+ coordinationv1listers "k8s.io/client-go/listers/coordination/v1"
corev1listers "k8s.io/client-go/listers/core/v1"
+ "k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
)
const (
// Workers is the number of workers for this controller
- Workers = 2
- ControllerName = "webhook-ca-controller"
- maxRetries = 10
+ Workers = 2
+ ControllerName = "webhook-controller"
+ DefaultWebhookTimeout = 10
+ AnnotationLastRequestTime = "kyverno.io/last-request-time"
+ IdleDeadline = tickerInterval * 5
+ maxRetries = 10
+ managedByLabel = "webhook.kyverno.io/managed-by"
+ tickerInterval = 30 * time.Second
+)
+
+var (
+ noneOnDryRun = admissionregistrationv1.SideEffectClassNoneOnDryRun
+ ifNeeded = admissionregistrationv1.IfNeededReinvocationPolicy
+ ignore = admissionregistrationv1.Ignore
+ fail = admissionregistrationv1.Fail
+ policyRule = admissionregistrationv1.Rule{
+ Resources: []string{"clusterpolicies/*", "policies/*"},
+ APIGroups: []string{"kyverno.io"},
+ APIVersions: []string{"v1", "v2beta1"},
+ }
+ verifyRule = admissionregistrationv1.Rule{
+ Resources: []string{"leases"},
+ APIGroups: []string{"coordination.k8s.io"},
+ APIVersions: []string{"v1"},
+ }
)
type controller struct {
// clients
- secretClient controllerutils.GetClient[*corev1.Secret]
- mwcClient controllerutils.UpdateClient[*admissionregistrationv1.MutatingWebhookConfiguration]
- vwcClient controllerutils.UpdateClient[*admissionregistrationv1.ValidatingWebhookConfiguration]
+ discoveryClient dclient.IDiscovery
+ secretClient controllerutils.GetClient[*corev1.Secret]
+ mwcClient controllerutils.ObjectClient[*admissionregistrationv1.MutatingWebhookConfiguration]
+ vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration]
+ leaseClient controllerutils.UpdateClient[*coordinationv1.Lease]
+ kyvernoClient versioned.Interface
// listers
- secretLister corev1listers.SecretLister
- configMapLister corev1listers.ConfigMapLister
mwcLister admissionregistrationv1listers.MutatingWebhookConfigurationLister
vwcLister admissionregistrationv1listers.ValidatingWebhookConfigurationLister
+ cpolLister kyvernov1listers.ClusterPolicyLister
+ polLister kyvernov1listers.PolicyLister
+ secretLister corev1listers.SecretLister
+ configMapLister corev1listers.ConfigMapLister
+ leaseLister coordinationv1listers.LeaseLister
// queue
- queue workqueue.RateLimitingInterface
- mwcEnqueue controllerutils.EnqueueFunc
- vwcEnqueue controllerutils.EnqueueFunc
+ queue workqueue.RateLimitingInterface
+
+ // config
+ server string
+ defaultTimeout int32
+ autoUpdateWebhooks bool
+ runtime runtimeutils.Runtime
+
+ // state
+ lock sync.RWMutex
+ policyState map[string]sets.String
}
func NewController(
+ discoveryClient dclient.IDiscovery,
secretClient controllerutils.GetClient[*corev1.Secret],
- mwcClient controllerutils.UpdateClient[*admissionregistrationv1.MutatingWebhookConfiguration],
- vwcClient controllerutils.UpdateClient[*admissionregistrationv1.ValidatingWebhookConfiguration],
- secretInformer corev1informers.SecretInformer,
- configMapInformer corev1informers.ConfigMapInformer,
+ mwcClient controllerutils.ObjectClient[*admissionregistrationv1.MutatingWebhookConfiguration],
+ vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration],
+ leaseClient controllerutils.UpdateClient[*coordinationv1.Lease],
+ kyvernoClient versioned.Interface,
mwcInformer admissionregistrationv1informers.MutatingWebhookConfigurationInformer,
vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer,
+ cpolInformer kyvernov1informers.ClusterPolicyInformer,
+ polInformer kyvernov1informers.PolicyInformer,
+ secretInformer corev1informers.SecretInformer,
+ configMapInformer corev1informers.ConfigMapInformer,
+ leaseInformer coordinationv1informers.LeaseInformer,
+ server string,
+ defaultTimeout int32,
+ autoUpdateWebhooks bool,
+ runtime runtimeutils.Runtime,
) controllers.Controller {
queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ControllerName)
c := controller{
- secretClient: secretClient,
- mwcClient: mwcClient,
- vwcClient: vwcClient,
- secretLister: secretInformer.Lister(),
- configMapLister: configMapInformer.Lister(),
- mwcLister: mwcInformer.Lister(),
- vwcLister: vwcInformer.Lister(),
- queue: queue,
- mwcEnqueue: controllerutils.AddDefaultEventHandlers(logger.V(3), mwcInformer.Informer(), queue),
- vwcEnqueue: controllerutils.AddDefaultEventHandlers(logger.V(3), vwcInformer.Informer(), queue),
+ discoveryClient: discoveryClient,
+ secretClient: secretClient,
+ mwcClient: mwcClient,
+ vwcClient: vwcClient,
+ leaseClient: leaseClient,
+ kyvernoClient: kyvernoClient,
+ mwcLister: mwcInformer.Lister(),
+ vwcLister: vwcInformer.Lister(),
+ cpolLister: cpolInformer.Lister(),
+ polLister: polInformer.Lister(),
+ secretLister: secretInformer.Lister(),
+ configMapLister: configMapInformer.Lister(),
+ leaseLister: leaseInformer.Lister(),
+ queue: queue,
+ server: server,
+ defaultTimeout: defaultTimeout,
+ autoUpdateWebhooks: autoUpdateWebhooks,
+ runtime: runtime,
+ policyState: map[string]sets.String{
+ config.MutatingWebhookConfigurationName: sets.NewString(),
+ config.ValidatingWebhookConfigurationName: sets.NewString(),
+ },
}
+ controllerutils.AddDefaultEventHandlers(logger, mwcInformer.Informer(), queue)
+ controllerutils.AddDefaultEventHandlers(logger, vwcInformer.Informer(), queue)
controllerutils.AddEventHandlersT(
secretInformer.Informer(),
- func(obj *corev1.Secret) { c.secretChanged(obj) },
- func(_, obj *corev1.Secret) { c.secretChanged(obj) },
- func(obj *corev1.Secret) { c.secretChanged(obj) },
+ func(obj *corev1.Secret) {
+ if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == tls.GenerateRootCASecretName() {
+ c.enqueueAll()
+ }
+ },
+ func(_, obj *corev1.Secret) {
+ if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == tls.GenerateRootCASecretName() {
+ c.enqueueAll()
+ }
+ },
+ func(obj *corev1.Secret) {
+ if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == tls.GenerateRootCASecretName() {
+ c.enqueueAll()
+ }
+ },
)
controllerutils.AddEventHandlersT(
configMapInformer.Informer(),
- func(obj *corev1.ConfigMap) { c.configMapChanged(obj) },
- func(_, obj *corev1.ConfigMap) { c.configMapChanged(obj) },
- func(obj *corev1.ConfigMap) { c.configMapChanged(obj) },
+ func(obj *corev1.ConfigMap) {
+ if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoConfigMapName() {
+ c.enqueueAll()
+ }
+ },
+ func(_, obj *corev1.ConfigMap) {
+ if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoConfigMapName() {
+ c.enqueueAll()
+ }
+ },
+ func(obj *corev1.ConfigMap) {
+ if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoConfigMapName() {
+ c.enqueueAll()
+ }
+ },
+ )
+ controllerutils.AddEventHandlers(
+ cpolInformer.Informer(),
+ func(interface{}) { c.enqueueResourceWebhooks(0) },
+ func(interface{}, interface{}) { c.enqueueResourceWebhooks(0) },
+ func(interface{}) { c.enqueueResourceWebhooks(0) },
+ )
+ controllerutils.AddEventHandlers(
+ polInformer.Informer(),
+ func(interface{}) { c.enqueueResourceWebhooks(0) },
+ func(interface{}, interface{}) { c.enqueueResourceWebhooks(0) },
+ func(interface{}) { c.enqueueResourceWebhooks(0) },
)
return &c
}
func (c *controller) Run(ctx context.Context, workers int) {
- controllerutils.Run(ctx, ControllerName, logger.V(3), c.queue, workers, maxRetries, c.reconcile)
+ // add our known webhooks to the queue
+ c.enqueueAll()
+ go c.watchdog(ctx)
+ controllerutils.Run(ctx, ControllerName, logger, c.queue, workers, maxRetries, c.reconcile)
}
-func (c *controller) secretChanged(secret *corev1.Secret) {
- if secret.GetName() == tls.GenerateRootCASecretName() && secret.GetNamespace() == config.KyvernoNamespace() {
- if err := c.enqueueAll(); err != nil {
- logger.Error(err, "failed to enqueue on secret change")
+func (c *controller) watchdog(ctx context.Context) {
+ ticker := time.NewTicker(tickerInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ lease, err := c.getLease()
+ if err != nil {
+ logger.Error(err, "failed to get lease")
+ } else {
+ if _, err := controllerutils.Update(
+ ctx,
+ lease,
+ c.leaseClient,
+ func(lease *coordinationv1.Lease) error {
+ if lease.Annotations == nil {
+ lease.Annotations = map[string]string{}
+ }
+ lease.Annotations[AnnotationLastRequestTime] = time.Now().Format(time.RFC3339)
+ if lease.Labels == nil {
+ lease.Labels = map[string]string{}
+ }
+ lease.Labels["app.kubernetes.io/name"] = kyvernov1.ValueKyvernoApp
+ return nil
+ },
+ ); err != nil {
+ logger.Error(err, "failed to get lease")
+ }
+ }
+ c.enqueueResourceWebhooks(0)
}
}
}
-func (c *controller) configMapChanged(cm *corev1.ConfigMap) {
- if cm.GetName() == config.KyvernoConfigMapName() && cm.GetNamespace() == config.KyvernoNamespace() {
- if err := c.enqueueAll(); err != nil {
- logger.Error(err, "failed to enqueue on configmap change")
- }
+func (c *controller) watchdogCheck() bool {
+ lease, err := c.getLease()
+ if err != nil {
+ logger.Error(err, "failed to get lease")
+ return false
}
+ annotations := lease.GetAnnotations()
+ if annotations == nil {
+ return false
+ }
+ annTime, err := time.Parse(time.RFC3339, annotations[AnnotationLastRequestTime])
+ if err != nil {
+ return false
+ }
+ return time.Now().Before(annTime.Add(IdleDeadline))
}
-func (c *controller) enqueueAll() error {
- requirement, err := labels.NewRequirement("webhook.kyverno.io/managed-by", selection.Equals, []string{kyvernov1.ValueKyvernoApp})
- if err != nil {
- return err
- }
- selector := labels.Everything().Add(*requirement)
- mwcs, err := c.mwcLister.List(selector)
- if err != nil {
- return err
- }
- for _, mwc := range mwcs {
- err = c.mwcEnqueue(mwc)
- if err != nil {
- return err
- }
- }
- vwcs, err := c.vwcLister.List(selector)
- if err != nil {
- return err
- }
- for _, vwc := range vwcs {
- err = c.vwcEnqueue(vwc)
- if err != nil {
- return err
- }
- }
- return nil
+func (c *controller) enqueueAll() {
+ c.enqueuePolicyWebhooks()
+ c.enqueueResourceWebhooks(0)
+ c.enqueueVerifyWebhook()
+}
+
+func (c *controller) enqueuePolicyWebhooks() {
+ c.queue.Add(config.PolicyValidatingWebhookConfigurationName)
+ c.queue.Add(config.PolicyMutatingWebhookConfigurationName)
+}
+
+func (c *controller) enqueueResourceWebhooks(duration time.Duration) {
+ c.queue.AddAfter(config.MutatingWebhookConfigurationName, duration)
+ c.queue.AddAfter(config.ValidatingWebhookConfigurationName, duration)
+}
+
+func (c *controller) enqueueVerifyWebhook() {
+ c.queue.Add(config.VerifyMutatingWebhookConfigurationName)
}
func (c *controller) loadConfig() config.Configuration {
- cfg := config.NewDefaultConfiguration(nil)
+ cfg := config.NewDefaultConfiguration()
cm, err := c.configMapLister.ConfigMaps(config.KyvernoNamespace()).Get(config.KyvernoConfigMapName())
if err == nil {
cfg.Load(cm)
@@ -141,78 +285,575 @@ func (c *controller) loadConfig() config.Configuration {
return cfg
}
-func (c *controller) reconcileMutatingWebhookConfiguration(ctx context.Context, logger logr.Logger, name string) error {
- w, err := c.mwcLister.Get(name)
- if err != nil {
- if apierrors.IsNotFound(err) {
- return nil
+func (c *controller) recordPolicyState(webhookConfigurationName string, policies ...kyvernov1.PolicyInterface) {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+ if _, ok := c.policyState[webhookConfigurationName]; !ok {
+ return
+ }
+ c.policyState[webhookConfigurationName] = sets.NewString()
+ for _, policy := range policies {
+ policyKey, err := cache.MetaNamespaceKeyFunc(policy)
+ if err != nil {
+ logger.Error(err, "failed to compute policy key", "policy", policy)
}
- return err
+ c.policyState[webhookConfigurationName].Insert(policyKey)
}
- labels := w.GetLabels()
- if labels == nil || labels["webhook.kyverno.io/managed-by"] != kyvernov1.ValueKyvernoApp {
- return nil
+}
+
+func (c *controller) clientConfig(caBundle []byte, path string) admissionregistrationv1.WebhookClientConfig {
+ clientConfig := admissionregistrationv1.WebhookClientConfig{
+ CABundle: caBundle,
}
- cfg := c.loadConfig()
- webhookCfg := config.WebhookConfig{}
- webhookCfgs := cfg.GetWebhooks()
- if len(webhookCfgs) > 0 {
- webhookCfg = webhookCfgs[0]
+ if c.server == "" {
+ clientConfig.Service = &admissionregistrationv1.ServiceReference{
+ Namespace: config.KyvernoNamespace(),
+ Name: config.KyvernoServiceName(),
+ Path: &path,
+ }
+ } else {
+ url := fmt.Sprintf("https://%s%s", c.server, path)
+ clientConfig.URL = &url
}
+ return clientConfig
+}
+
+func (c *controller) reconcileResourceValidatingWebhookConfiguration(ctx context.Context) error {
+ if c.autoUpdateWebhooks {
+ return c.reconcileValidatingWebhookConfiguration(ctx, c.autoUpdateWebhooks, c.buildResourceValidatingWebhookConfiguration)
+ } else {
+ return c.reconcileValidatingWebhookConfiguration(ctx, c.autoUpdateWebhooks, c.buildDefaultResourceValidatingWebhookConfiguration)
+ }
+}
+
+func (c *controller) reconcileResourceMutatingWebhookConfiguration(ctx context.Context) error {
+ if c.autoUpdateWebhooks {
+ return c.reconcileMutatingWebhookConfiguration(ctx, c.autoUpdateWebhooks, c.buildResourceMutatingWebhookConfiguration)
+ } else {
+ return c.reconcileMutatingWebhookConfiguration(ctx, c.autoUpdateWebhooks, c.buildDefaultResourceMutatingWebhookConfiguration)
+ }
+}
+
+func (c *controller) reconcilePolicyValidatingWebhookConfiguration(ctx context.Context) error {
+ return c.reconcileValidatingWebhookConfiguration(ctx, true, c.buildPolicyValidatingWebhookConfiguration)
+}
+
+func (c *controller) reconcilePolicyMutatingWebhookConfiguration(ctx context.Context) error {
+ return c.reconcileMutatingWebhookConfiguration(ctx, true, c.buildPolicyMutatingWebhookConfiguration)
+}
+
+func (c *controller) reconcileVerifyMutatingWebhookConfiguration(ctx context.Context) error {
+ return c.reconcileMutatingWebhookConfiguration(ctx, true, c.buildVerifyMutatingWebhookConfiguration)
+}
+
+func (c *controller) reconcileValidatingWebhookConfiguration(ctx context.Context, autoUpdateWebhooks bool, build func([]byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error)) error {
caData, err := tls.ReadRootCASecret(c.secretClient)
if err != nil {
return err
}
- _, err = controllerutils.Update(ctx, w, c.mwcClient, func(w *admissionregistrationv1.MutatingWebhookConfiguration) error {
- for i := range w.Webhooks {
- w.Webhooks[i].ClientConfig.CABundle = caData
- w.Webhooks[i].ObjectSelector = webhookCfg.ObjectSelector
- w.Webhooks[i].NamespaceSelector = webhookCfg.NamespaceSelector
+ desired, err := build(caData)
+ if err != nil {
+ return err
+ }
+ observed, err := c.vwcLister.Get(desired.Name)
+ if err != nil {
+ if apierrors.IsNotFound(err) {
+ _, err := c.vwcClient.Create(ctx, desired, metav1.CreateOptions{})
+ return err
}
+ return err
+ }
+ if !autoUpdateWebhooks {
+ return nil
+ }
+ _, err = controllerutils.Update(ctx, observed, c.vwcClient, func(w *admissionregistrationv1.ValidatingWebhookConfiguration) error {
+ w.Labels = desired.Labels
+ w.OwnerReferences = desired.OwnerReferences
+ w.Webhooks = desired.Webhooks
return nil
})
return err
}
-func (c *controller) reconcileValidatingWebhookConfiguration(ctx context.Context, logger logr.Logger, name string) error {
- w, err := c.vwcLister.Get(name)
- if err != nil {
- if apierrors.IsNotFound(err) {
- return nil
- }
- return err
- }
- labels := w.GetLabels()
- if labels == nil || labels["webhook.kyverno.io/managed-by"] != kyvernov1.ValueKyvernoApp {
- return nil
- }
- cfg := c.loadConfig()
- webhookCfg := config.WebhookConfig{}
- webhookCfgs := cfg.GetWebhooks()
- if len(webhookCfgs) > 0 {
- webhookCfg = webhookCfgs[0]
- }
+func (c *controller) reconcileMutatingWebhookConfiguration(ctx context.Context, autoUpdateWebhooks bool, build func([]byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error)) error {
caData, err := tls.ReadRootCASecret(c.secretClient)
if err != nil {
return err
}
- _, err = controllerutils.Update(ctx, w, c.vwcClient, func(w *admissionregistrationv1.ValidatingWebhookConfiguration) error {
- for i := range w.Webhooks {
- w.Webhooks[i].ClientConfig.CABundle = caData
- w.Webhooks[i].ObjectSelector = webhookCfg.ObjectSelector
- w.Webhooks[i].NamespaceSelector = webhookCfg.NamespaceSelector
+ desired, err := build(caData)
+ if err != nil {
+ return err
+ }
+ observed, err := c.mwcLister.Get(desired.Name)
+ if err != nil {
+ if apierrors.IsNotFound(err) {
+ _, err := c.mwcClient.Create(ctx, desired, metav1.CreateOptions{})
+ return err
}
+ return err
+ }
+ if !autoUpdateWebhooks {
+ return nil
+ }
+ _, err = controllerutils.Update(ctx, observed, c.mwcClient, func(w *admissionregistrationv1.MutatingWebhookConfiguration) error {
+ w.Labels = desired.Labels
+ w.OwnerReferences = desired.OwnerReferences
+ w.Webhooks = desired.Webhooks
return nil
})
return err
}
-func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, namespace, name string) error {
- if err := c.reconcileMutatingWebhookConfiguration(ctx, logger, name); err != nil {
+func (c *controller) updatePolicyStatuses(ctx context.Context) error {
+ c.lock.RLock()
+ defer c.lock.RUnlock()
+ policies, err := c.getAllPolicies()
+ if err != nil {
return err
}
- if err := c.reconcileValidatingWebhookConfiguration(ctx, logger, name); err != nil {
- return err
+ for _, policy := range policies {
+ policyKey, err := cache.MetaNamespaceKeyFunc(policy)
+ if err != nil {
+ return err
+ }
+ ready := true
+ for _, set := range c.policyState {
+ if !set.Has(policyKey) {
+ ready = false
+ }
+ }
+ if policy.IsReady() != ready {
+ policy = policy.CreateDeepCopy()
+ status := policy.GetStatus()
+ status.SetReady(ready)
+ if policy.GetNamespace() == "" {
+ _, err := c.kyvernoClient.KyvernoV1().ClusterPolicies().UpdateStatus(ctx, policy.(*kyvernov1.ClusterPolicy), metav1.UpdateOptions{})
+ if err != nil {
+ return err
+ }
+ } else {
+ _, err := c.kyvernoClient.KyvernoV1().Policies(policy.GetNamespace()).UpdateStatus(ctx, policy.(*kyvernov1.Policy), metav1.UpdateOptions{})
+ if err != nil {
+ return err
+ }
+ }
+ }
}
return nil
}
+
+func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, namespace, name string) error {
+ switch name {
+ case config.MutatingWebhookConfigurationName:
+ if c.runtime.IsRollingUpdate() {
+ c.enqueueResourceWebhooks(1 * time.Second)
+ } else {
+ if err := c.reconcileResourceMutatingWebhookConfiguration(ctx); err != nil {
+ return err
+ }
+ if err := c.updatePolicyStatuses(ctx); err != nil {
+ return err
+ }
+ }
+ case config.ValidatingWebhookConfigurationName:
+ if c.runtime.IsRollingUpdate() {
+ c.enqueueResourceWebhooks(1 * time.Second)
+ } else {
+ if err := c.reconcileResourceValidatingWebhookConfiguration(ctx); err != nil {
+ return err
+ }
+ if err := c.updatePolicyStatuses(ctx); err != nil {
+ return err
+ }
+ }
+ case config.PolicyValidatingWebhookConfigurationName:
+ return c.reconcilePolicyValidatingWebhookConfiguration(ctx)
+ case config.PolicyMutatingWebhookConfigurationName:
+ return c.reconcilePolicyMutatingWebhookConfiguration(ctx)
+ case config.VerifyMutatingWebhookConfigurationName:
+ return c.reconcileVerifyMutatingWebhookConfiguration(ctx)
+ }
+ return nil
+}
+
+func (c *controller) buildVerifyMutatingWebhookConfiguration(caBundle []byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
+ return &admissionregistrationv1.MutatingWebhookConfiguration{
+ ObjectMeta: objectMeta(config.VerifyMutatingWebhookConfigurationName),
+ Webhooks: []admissionregistrationv1.MutatingWebhook{{
+ Name: config.VerifyMutatingWebhookName,
+ ClientConfig: c.clientConfig(caBundle, config.VerifyMutatingWebhookServicePath),
+ Rules: []admissionregistrationv1.RuleWithOperations{{
+ Rule: verifyRule,
+ Operations: []admissionregistrationv1.OperationType{
+ admissionregistrationv1.Update,
+ },
+ }},
+ FailurePolicy: &ignore,
+ SideEffects: &noneOnDryRun,
+ ReinvocationPolicy: &ifNeeded,
+ AdmissionReviewVersions: []string{"v1beta1"},
+ ObjectSelector: &metav1.LabelSelector{
+ MatchLabels: map[string]string{
+ "app.kubernetes.io/name": kyvernov1.ValueKyvernoApp,
+ },
+ },
+ }},
+ },
+ nil
+}
+
+func (c *controller) buildPolicyMutatingWebhookConfiguration(caBundle []byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
+ return &admissionregistrationv1.MutatingWebhookConfiguration{
+ ObjectMeta: objectMeta(config.PolicyMutatingWebhookConfigurationName),
+ Webhooks: []admissionregistrationv1.MutatingWebhook{{
+ Name: config.PolicyMutatingWebhookName,
+ ClientConfig: c.clientConfig(caBundle, config.PolicyMutatingWebhookServicePath),
+ Rules: []admissionregistrationv1.RuleWithOperations{{
+ Rule: policyRule,
+ Operations: []admissionregistrationv1.OperationType{
+ admissionregistrationv1.Create,
+ admissionregistrationv1.Update,
+ },
+ }},
+ FailurePolicy: &ignore,
+ SideEffects: &noneOnDryRun,
+ ReinvocationPolicy: &ifNeeded,
+ AdmissionReviewVersions: []string{"v1beta1"},
+ }},
+ },
+ nil
+}
+
+func (c *controller) buildPolicyValidatingWebhookConfiguration(caBundle []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) {
+ return &admissionregistrationv1.ValidatingWebhookConfiguration{
+ ObjectMeta: objectMeta(config.PolicyValidatingWebhookConfigurationName),
+ Webhooks: []admissionregistrationv1.ValidatingWebhook{{
+ Name: config.PolicyValidatingWebhookName,
+ ClientConfig: c.clientConfig(caBundle, config.PolicyValidatingWebhookServicePath),
+ Rules: []admissionregistrationv1.RuleWithOperations{{
+ Rule: policyRule,
+ Operations: []admissionregistrationv1.OperationType{
+ admissionregistrationv1.Create,
+ admissionregistrationv1.Update,
+ },
+ }},
+ FailurePolicy: &ignore,
+ SideEffects: &noneOnDryRun,
+ AdmissionReviewVersions: []string{"v1beta1"},
+ }},
+ },
+ nil
+}
+
+func (c *controller) buildDefaultResourceMutatingWebhookConfiguration(caBundle []byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
+ return &admissionregistrationv1.MutatingWebhookConfiguration{
+ ObjectMeta: objectMeta(config.MutatingWebhookConfigurationName),
+ Webhooks: []admissionregistrationv1.MutatingWebhook{{
+ Name: config.MutatingWebhookName + "-ignore",
+ ClientConfig: c.clientConfig(caBundle, config.MutatingWebhookServicePath+"/ignore"),
+ Rules: []admissionregistrationv1.RuleWithOperations{{
+ Rule: admissionregistrationv1.Rule{
+ APIGroups: []string{"*"},
+ APIVersions: []string{"*"},
+ Resources: []string{"*/*"},
+ },
+ Operations: []admissionregistrationv1.OperationType{
+ admissionregistrationv1.Create,
+ admissionregistrationv1.Update,
+ },
+ }},
+ FailurePolicy: &ignore,
+ SideEffects: &noneOnDryRun,
+ AdmissionReviewVersions: []string{"v1beta1"},
+ TimeoutSeconds: &c.defaultTimeout,
+ ReinvocationPolicy: &ifNeeded,
+ }},
+ },
+ nil
+}
+
+func (c *controller) buildResourceMutatingWebhookConfiguration(caBundle []byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
+ result := admissionregistrationv1.MutatingWebhookConfiguration{
+ ObjectMeta: objectMeta(config.MutatingWebhookConfigurationName),
+ Webhooks: []admissionregistrationv1.MutatingWebhook{},
+ }
+ if c.watchdogCheck() {
+ ignore := newWebhook(c.defaultTimeout, ignore)
+ fail := newWebhook(c.defaultTimeout, fail)
+ policies, err := c.getAllPolicies()
+ if err != nil {
+ return nil, err
+ }
+ c.recordPolicyState(config.MutatingWebhookConfigurationName, policies...)
+ // TODO: shouldn't be per failure policy, depending of the policy/rules that apply ?
+ if hasWildcard(policies...) {
+ ignore.setWildcard()
+ fail.setWildcard()
+ } else {
+ for _, p := range policies {
+ spec := p.GetSpec()
+ if spec.HasMutate() || spec.HasVerifyImages() {
+ if spec.GetFailurePolicy() == kyvernov1.Ignore {
+ c.mergeWebhook(ignore, p, false)
+ } else {
+ c.mergeWebhook(fail, p, false)
+ }
+ }
+ }
+ }
+ cfg := c.loadConfig()
+ webhookCfg := config.WebhookConfig{}
+ webhookCfgs := cfg.GetWebhooks()
+ if len(webhookCfgs) > 0 {
+ webhookCfg = webhookCfgs[0]
+ }
+ if !ignore.isEmpty() {
+ result.Webhooks = append(
+ result.Webhooks,
+ admissionregistrationv1.MutatingWebhook{
+ Name: config.MutatingWebhookName + "-ignore",
+ ClientConfig: c.clientConfig(caBundle, config.MutatingWebhookServicePath+"/ignore"),
+ Rules: []admissionregistrationv1.RuleWithOperations{
+ ignore.buildRuleWithOperations(admissionregistrationv1.Create, admissionregistrationv1.Update),
+ },
+ FailurePolicy: &ignore.failurePolicy,
+ SideEffects: &noneOnDryRun,
+ AdmissionReviewVersions: []string{"v1beta1"},
+ NamespaceSelector: webhookCfg.NamespaceSelector,
+ ObjectSelector: webhookCfg.ObjectSelector,
+ TimeoutSeconds: &ignore.maxWebhookTimeout,
+ ReinvocationPolicy: &ifNeeded,
+ },
+ )
+ }
+ if !fail.isEmpty() {
+ result.Webhooks = append(
+ result.Webhooks,
+ admissionregistrationv1.MutatingWebhook{
+ Name: config.MutatingWebhookName + "-fail",
+ ClientConfig: c.clientConfig(caBundle, config.MutatingWebhookServicePath+"/fail"),
+ Rules: []admissionregistrationv1.RuleWithOperations{
+ fail.buildRuleWithOperations(admissionregistrationv1.Create, admissionregistrationv1.Update),
+ },
+ FailurePolicy: &fail.failurePolicy,
+ SideEffects: &noneOnDryRun,
+ AdmissionReviewVersions: []string{"v1beta1"},
+ NamespaceSelector: webhookCfg.NamespaceSelector,
+ ObjectSelector: webhookCfg.ObjectSelector,
+ TimeoutSeconds: &fail.maxWebhookTimeout,
+ ReinvocationPolicy: &ifNeeded,
+ },
+ )
+ }
+ } else {
+ c.recordPolicyState(config.MutatingWebhookConfigurationName)
+ }
+ return &result, nil
+}
+
+func (c *controller) buildDefaultResourceValidatingWebhookConfiguration(caBundle []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) {
+ return &admissionregistrationv1.ValidatingWebhookConfiguration{
+ ObjectMeta: objectMeta(config.ValidatingWebhookConfigurationName),
+ Webhooks: []admissionregistrationv1.ValidatingWebhook{{
+ Name: config.ValidatingWebhookName + "-ignore",
+ ClientConfig: c.clientConfig(caBundle, config.ValidatingWebhookServicePath+"/ignore"),
+ Rules: []admissionregistrationv1.RuleWithOperations{{
+ Rule: admissionregistrationv1.Rule{
+ APIGroups: []string{"*"},
+ APIVersions: []string{"*"},
+ Resources: []string{"*/*"},
+ },
+ Operations: []admissionregistrationv1.OperationType{
+ admissionregistrationv1.Create,
+ admissionregistrationv1.Update,
+ admissionregistrationv1.Delete,
+ admissionregistrationv1.Connect,
+ },
+ }},
+ FailurePolicy: &ignore,
+ SideEffects: &noneOnDryRun,
+ AdmissionReviewVersions: []string{"v1beta1"},
+ TimeoutSeconds: &c.defaultTimeout,
+ }},
+ },
+ nil
+}
+
+func (c *controller) buildResourceValidatingWebhookConfiguration(caBundle []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) {
+ result := admissionregistrationv1.ValidatingWebhookConfiguration{
+ ObjectMeta: objectMeta(config.ValidatingWebhookConfigurationName),
+ Webhooks: []admissionregistrationv1.ValidatingWebhook{},
+ }
+ if c.watchdogCheck() {
+ ignore := newWebhook(c.defaultTimeout, ignore)
+ fail := newWebhook(c.defaultTimeout, fail)
+ policies, err := c.getAllPolicies()
+ if err != nil {
+ return nil, err
+ }
+ c.recordPolicyState(config.ValidatingWebhookConfigurationName, policies...)
+ // TODO: shouldn't be per failure policy, depending of the policy/rules that apply ?
+ if hasWildcard(policies...) {
+ ignore.setWildcard()
+ fail.setWildcard()
+ } else {
+ for _, p := range policies {
+ spec := p.GetSpec()
+ if spec.HasValidate() || spec.HasGenerate() || spec.HasMutate() || spec.HasImagesValidationChecks() || spec.HasYAMLSignatureVerify() {
+ if spec.GetFailurePolicy() == kyvernov1.Ignore {
+ c.mergeWebhook(ignore, p, true)
+ } else {
+ c.mergeWebhook(fail, p, true)
+ }
+ }
+ }
+ }
+ cfg := c.loadConfig()
+ webhookCfg := config.WebhookConfig{}
+ webhookCfgs := cfg.GetWebhooks()
+ if len(webhookCfgs) > 0 {
+ webhookCfg = webhookCfgs[0]
+ }
+ if !ignore.isEmpty() {
+ result.Webhooks = append(
+ result.Webhooks,
+ admissionregistrationv1.ValidatingWebhook{
+ Name: config.ValidatingWebhookName + "-ignore",
+ ClientConfig: c.clientConfig(caBundle, config.ValidatingWebhookServicePath+"/ignore"),
+ Rules: []admissionregistrationv1.RuleWithOperations{
+ ignore.buildRuleWithOperations(admissionregistrationv1.Create, admissionregistrationv1.Update, admissionregistrationv1.Delete, admissionregistrationv1.Connect),
+ },
+ FailurePolicy: &ignore.failurePolicy,
+ SideEffects: &noneOnDryRun,
+ AdmissionReviewVersions: []string{"v1beta1"},
+ NamespaceSelector: webhookCfg.NamespaceSelector,
+ ObjectSelector: webhookCfg.ObjectSelector,
+ TimeoutSeconds: &ignore.maxWebhookTimeout,
+ },
+ )
+ }
+ if !fail.isEmpty() {
+ result.Webhooks = append(
+ result.Webhooks,
+ admissionregistrationv1.ValidatingWebhook{
+ Name: config.ValidatingWebhookName + "-fail",
+ ClientConfig: c.clientConfig(caBundle, config.ValidatingWebhookServicePath+"/fail"),
+ Rules: []admissionregistrationv1.RuleWithOperations{
+ fail.buildRuleWithOperations(admissionregistrationv1.Create, admissionregistrationv1.Update, admissionregistrationv1.Delete, admissionregistrationv1.Connect),
+ },
+ FailurePolicy: &fail.failurePolicy,
+ SideEffects: &noneOnDryRun,
+ AdmissionReviewVersions: []string{"v1beta1"},
+ NamespaceSelector: webhookCfg.NamespaceSelector,
+ ObjectSelector: webhookCfg.ObjectSelector,
+ TimeoutSeconds: &fail.maxWebhookTimeout,
+ },
+ )
+ }
+ } else {
+ c.recordPolicyState(config.MutatingWebhookConfigurationName)
+ }
+ return &result, nil
+}
+
+func (c *controller) getAllPolicies() ([]kyvernov1.PolicyInterface, error) {
+ var policies []kyvernov1.PolicyInterface
+ if cpols, err := c.cpolLister.List(labels.Everything()); err != nil {
+ return nil, err
+ } else {
+ for _, cpol := range cpols {
+ policies = append(policies, cpol)
+ }
+ }
+ if pols, err := c.polLister.List(labels.Everything()); err != nil {
+ return nil, err
+ } else {
+ for _, pol := range pols {
+ policies = append(policies, pol)
+ }
+ }
+ return policies, nil
+}
+
+func (c *controller) getLease() (*coordinationv1.Lease, error) {
+ return c.leaseLister.Leases(config.KyvernoNamespace()).Get("kyverno")
+}
+
+// mergeWebhook merges the matching kinds of the policy to webhook.rule
+func (c *controller) mergeWebhook(dst *webhook, policy kyvernov1.PolicyInterface, updateValidate bool) {
+ matchedGVK := make([]string, 0)
+ for _, rule := range autogen.ComputeRules(policy) {
+ // matching kinds in generate policies need to be added to both webhook
+ if rule.HasGenerate() {
+ matchedGVK = append(matchedGVK, rule.MatchResources.GetKinds()...)
+ matchedGVK = append(matchedGVK, rule.Generation.ResourceSpec.Kind)
+ continue
+ }
+ if (updateValidate && rule.HasValidate() || rule.HasImagesValidationChecks()) ||
+ (updateValidate && rule.HasMutate() && rule.IsMutateExisting()) ||
+ (!updateValidate && rule.HasMutate()) && !rule.IsMutateExisting() ||
+ (!updateValidate && rule.HasVerifyImages()) || (!updateValidate && rule.HasYAMLSignatureVerify()) {
+ matchedGVK = append(matchedGVK, rule.MatchResources.GetKinds()...)
+ }
+ }
+ gvkMap := make(map[string]int)
+ gvrList := make([]schema.GroupVersionResource, 0)
+ for _, gvk := range matchedGVK {
+ if _, ok := gvkMap[gvk]; !ok {
+ gvkMap[gvk] = 1
+ // NOTE: webhook stores GVR in its rules while policy stores GVK in its rules definition
+ gv, k := kubeutils.GetKindFromGVK(gvk)
+ switch k {
+ case "Binding":
+ gvrList = append(gvrList, schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods/binding"})
+ case "NodeProxyOptions":
+ gvrList = append(gvrList, schema.GroupVersionResource{Group: "", Version: "v1", Resource: "nodes/proxy"})
+ case "PodAttachOptions":
+ gvrList = append(gvrList, schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods/attach"})
+ case "PodExecOptions":
+ gvrList = append(gvrList, schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods/exec"})
+ case "PodPortForwardOptions":
+ gvrList = append(gvrList, schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods/portforward"})
+ case "PodProxyOptions":
+ gvrList = append(gvrList, schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods/proxy"})
+ case "ServiceProxyOptions":
+ gvrList = append(gvrList, schema.GroupVersionResource{Group: "", Version: "v1", Resource: "services/proxy"})
+ default:
+ _, gvr, err := c.discoveryClient.FindResource(gv, k)
+ if err != nil {
+ logger.Error(err, "unable to convert GVK to GVR", "GVK", gvk)
+ continue
+ }
+ if strings.Contains(gvk, "*") {
+ group := kubeutils.GetGroupFromGVK(gvk)
+ gvrList = append(gvrList, schema.GroupVersionResource{Group: group, Version: "*", Resource: gvr.Resource})
+ } else {
+ logger.V(4).Info("configuring webhook", "GVK", gvk, "GVR", gvr)
+ gvrList = append(gvrList, gvr)
+ }
+ }
+ }
+ }
+ for _, gvr := range gvrList {
+ dst.groups.Insert(gvr.Group)
+ if gvr.Version == "*" {
+ dst.versions = sets.NewString()
+ dst.versions.Insert(gvr.Version)
+ } else if !dst.versions.Has("*") {
+ dst.versions.Insert(gvr.Version)
+ }
+ dst.resources.Insert(gvr.Resource)
+ }
+ if dst.resources.Has("pods") {
+ dst.resources.Insert("pods/ephemeralcontainers")
+ }
+ if dst.resources.Has("services") {
+ dst.resources.Insert("services/status")
+ }
+ spec := policy.GetSpec()
+ if spec.WebhookTimeoutSeconds != nil {
+ if dst.maxWebhookTimeout < *spec.WebhookTimeoutSeconds {
+ dst.maxWebhookTimeout = *spec.WebhookTimeoutSeconds
+ }
+ }
+}
diff --git a/pkg/controllers/webhook/log.go b/pkg/controllers/webhook/log.go
index 8122928705..4bcd1d9e65 100644
--- a/pkg/controllers/webhook/log.go
+++ b/pkg/controllers/webhook/log.go
@@ -1,4 +1,4 @@
-package background
+package webhook
import "sigs.k8s.io/controller-runtime/pkg/log"
diff --git a/pkg/controllers/webhook/utils.go b/pkg/controllers/webhook/utils.go
new file mode 100644
index 0000000000..dcecff803b
--- /dev/null
+++ b/pkg/controllers/webhook/utils.go
@@ -0,0 +1,72 @@
+package webhook
+
+import (
+ kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
+ "github.com/kyverno/kyverno/pkg/utils"
+ admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/util/sets"
+)
+
+// webhook is the instance that aggregates the GVK of existing policies
+// based on kind, failurePolicy and webhookTimeout
+type webhook struct {
+ maxWebhookTimeout int32
+ failurePolicy admissionregistrationv1.FailurePolicyType
+ groups sets.String
+ versions sets.String
+ resources sets.String
+}
+
+func newWebhook(timeout int32, failurePolicy admissionregistrationv1.FailurePolicyType) *webhook {
+ return &webhook{
+ maxWebhookTimeout: timeout,
+ failurePolicy: failurePolicy,
+ groups: sets.NewString(),
+ versions: sets.NewString(),
+ resources: sets.NewString(),
+ }
+}
+
+func (wh *webhook) buildRuleWithOperations(ops ...admissionregistrationv1.OperationType) admissionregistrationv1.RuleWithOperations {
+ return admissionregistrationv1.RuleWithOperations{
+ Rule: admissionregistrationv1.Rule{
+ APIGroups: wh.groups.List(),
+ APIVersions: wh.versions.List(),
+ Resources: wh.resources.List(),
+ },
+ Operations: ops,
+ }
+}
+
+func (wh *webhook) isEmpty() bool {
+ return wh.groups.Len() == 0 || wh.versions.Len() == 0 || wh.resources.Len() == 0
+}
+
+func (wh *webhook) setWildcard() {
+ wh.groups = sets.NewString("*")
+ wh.versions = sets.NewString("*")
+ wh.resources = sets.NewString("*/*")
+}
+
+func hasWildcard(policies ...kyvernov1.PolicyInterface) bool {
+ for _, policy := range policies {
+ spec := policy.GetSpec()
+ for _, rule := range spec.Rules {
+ if kinds := rule.MatchResources.GetKinds(); utils.ContainsString(kinds, "*") {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func objectMeta(name string, owner ...metav1.OwnerReference) metav1.ObjectMeta {
+ return metav1.ObjectMeta{
+ Name: name,
+ Labels: map[string]string{
+ managedByLabel: kyvernov1.ValueKyvernoApp,
+ },
+ OwnerReferences: owner,
+ }
+}
diff --git a/pkg/controllers/webhook/utils_test.go b/pkg/controllers/webhook/utils_test.go
new file mode 100644
index 0000000000..97cb03f12b
--- /dev/null
+++ b/pkg/controllers/webhook/utils_test.go
@@ -0,0 +1,16 @@
+package webhook
+
+import (
+ "testing"
+
+ "gotest.tools/assert"
+ admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+)
+
+func Test_webhook_isEmpty(t *testing.T) {
+ empty := newWebhook(DefaultWebhookTimeout, admissionregistrationv1.Ignore)
+ assert.Equal(t, empty.isEmpty(), true)
+ notEmpty := newWebhook(DefaultWebhookTimeout, admissionregistrationv1.Ignore)
+ notEmpty.setWildcard()
+ assert.Equal(t, notEmpty.isEmpty(), false)
+}
diff --git a/pkg/utils/runtime/utils.go b/pkg/utils/runtime/utils.go
new file mode 100644
index 0000000000..1367011ee7
--- /dev/null
+++ b/pkg/utils/runtime/utils.go
@@ -0,0 +1,121 @@
+package runtime
+
+import (
+ "time"
+
+ "github.com/go-logr/logr"
+ "github.com/kyverno/kyverno/pkg/config"
+ appsv1 "k8s.io/api/apps/v1"
+ coordinationv1 "k8s.io/api/coordination/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ appsv1informers "k8s.io/client-go/informers/apps/v1"
+ coordinationv1informers "k8s.io/client-go/informers/coordination/v1"
+ appsv1listers "k8s.io/client-go/listers/apps/v1"
+ coordinationv1listers "k8s.io/client-go/listers/coordination/v1"
+)
+
+const (
+ AnnotationLastRequestTime = "kyverno.io/last-request-time"
+ IdleDeadline = tickerInterval * 5
+ tickerInterval = 30 * time.Second
+)
+
+type Runtime interface {
+ IsDebug() bool
+ IsReady() bool
+ IsLive() bool
+ IsRollingUpdate() bool
+ IsGoingDown() bool
+}
+
+type runtime struct {
+ serverIP string
+ leaseLister coordinationv1listers.LeaseLister
+ deploymentLister appsv1listers.DeploymentLister
+ logger logr.Logger
+}
+
+func NewRuntime(
+ logger logr.Logger,
+ serverIP string,
+ leaseInformer coordinationv1informers.LeaseInformer,
+ deploymentInformer appsv1informers.DeploymentInformer,
+) Runtime {
+ return &runtime{
+ serverIP: serverIP,
+ leaseLister: leaseInformer.Lister(),
+ deploymentLister: deploymentInformer.Lister(),
+ logger: logger,
+ }
+}
+
+func (c *runtime) IsDebug() bool {
+ return c.serverIP != ""
+}
+
+func (c *runtime) IsLive() bool {
+ return c.IsDebug() || c.check()
+}
+
+func (c *runtime) IsReady() bool {
+ return c.IsDebug() || c.check()
+}
+
+func (c *runtime) IsRollingUpdate() bool {
+ if c.IsDebug() {
+ return false
+ }
+ deployment, err := c.getDeployment()
+ if err != nil {
+ return true
+ }
+ var replicas int32 = 1
+ if deployment.Spec.Replicas != nil {
+ replicas = *deployment.Spec.Replicas
+ }
+ nonTerminatedReplicas := deployment.Status.Replicas
+ if nonTerminatedReplicas > replicas {
+ c.logger.Info("detect Kyverno is in rolling update, won't trigger the update again")
+ return true
+ }
+ return false
+}
+
+func (c *runtime) IsGoingDown() bool {
+ if c.IsDebug() {
+ return false
+ }
+ deployment, err := c.getDeployment()
+ if err != nil {
+ return apierrors.IsNotFound(err)
+ }
+ if deployment.Spec.Replicas != nil {
+ return *deployment.Spec.Replicas == 0
+ }
+ return false
+}
+
+func (c *runtime) getLease() (*coordinationv1.Lease, error) {
+ return c.leaseLister.Leases(config.KyvernoNamespace()).Get("kyverno")
+}
+
+func (c *runtime) getDeployment() (*appsv1.Deployment, error) {
+ return c.deploymentLister.Deployments(config.KyvernoNamespace()).Get("kyverno")
+}
+
+func (c *runtime) check() bool {
+ lease, err := c.getLease()
+ if err != nil {
+ c.logger.Error(err, "failed to get lease")
+ return false
+ }
+ annotations := lease.GetAnnotations()
+ if annotations == nil {
+ return false
+ }
+ annTime, err := time.Parse(time.RFC3339, annotations[AnnotationLastRequestTime])
+ if err != nil {
+ return false
+ }
+ return time.Now().Before(annTime.Add(IdleDeadline))
+}
diff --git a/pkg/webhookconfig/common.go b/pkg/webhookconfig/common.go
deleted file mode 100644
index 03bca9f1bb..0000000000
--- a/pkg/webhookconfig/common.go
+++ /dev/null
@@ -1,383 +0,0 @@
-package webhookconfig
-
-import (
- "context"
- "errors"
- "fmt"
- "reflect"
- "strings"
-
- kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
- "github.com/kyverno/kyverno/pkg/config"
- "github.com/kyverno/kyverno/pkg/metrics"
- "github.com/kyverno/kyverno/pkg/tls"
- admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
- appsv1 "k8s.io/api/apps/v1"
- corev1 "k8s.io/api/core/v1"
- rbacv1 "k8s.io/api/rbac/v1"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-const (
- managedByLabel string = "webhook.kyverno.io/managed-by"
-)
-
-var (
- noneOnDryRun = admissionregistrationv1.SideEffectClassNoneOnDryRun
- never = admissionregistrationv1.NeverReinvocationPolicy
- ifNeeded = admissionregistrationv1.IfNeededReinvocationPolicy
- policyRule = admissionregistrationv1.Rule{
- Resources: []string{"clusterpolicies/*", "policies/*"},
- APIGroups: []string{"kyverno.io"},
- APIVersions: []string{"v1", "v2beta1"},
- }
- verifyRule = admissionregistrationv1.Rule{
- Resources: []string{"leases"},
- APIGroups: []string{"coordination.k8s.io"},
- APIVersions: []string{"v1"},
- }
- vertifyObjectSelector = &metav1.LabelSelector{
- MatchLabels: map[string]string{
- "app.kubernetes.io/name": kyvernov1.ValueKyvernoApp,
- },
- }
- update = []admissionregistrationv1.OperationType{admissionregistrationv1.Update}
- createUpdate = []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update}
- all = []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update, admissionregistrationv1.Delete, admissionregistrationv1.Connect}
-)
-
-func (wrc *Register) readCaData() []byte {
- logger := wrc.log.WithName("readCaData")
- var caData []byte
- var err error
- recorder := metrics.NamespacedClientQueryRecorder(wrc.metricsConfig, config.KyvernoNamespace(), "Secret", metrics.KubeClient)
- secretsClient := metrics.ObjectClient[*corev1.Secret](recorder, wrc.kubeClient.CoreV1().Secrets(config.KyvernoNamespace()))
- // Check if ca is defined in the secret tls-ca
- // assume the key and signed cert have been defined in secret tls.kyverno
- if caData, err = tls.ReadRootCASecret(secretsClient); err == nil {
- logger.V(4).Info("read CA from secret")
- return caData
- }
-
- logger.V(4).Info("failed to read CA from kubeconfig")
- return nil
-}
-
-func getHealthyPodsIP(pods []corev1.Pod) []string {
- var ips []string
- for _, pod := range pods {
- if pod.Status.Phase == "Running" {
- ips = append(ips, pod.Status.PodIP)
- }
- }
- return ips
-}
-
-func (wrc *Register) GetKubePolicyClusterRoleName() (*rbacv1.ClusterRole, error) {
- selector := &metav1.LabelSelector{
- MatchLabels: map[string]string{
- "app.kubernetes.io/name": kyvernov1.ValueKyvernoApp,
- },
- }
- clusterRoles, err := wrc.kubeClient.RbacV1().ClusterRoles().List(context.TODO(), metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)})
- wrc.metricsConfig.RecordClientQueries(metrics.ClientList, metrics.KubeClient, "ClusterRole", "")
- if err != nil {
- return nil, err
- }
- for _, cr := range clusterRoles.Items {
- if strings.HasSuffix(cr.GetName(), "webhook") {
- return &cr, nil
- }
- }
- return nil, errors.New("failed to get cluster role with suffix webhook")
-}
-
-// GetKubePolicyDeployment gets Kyverno deployment using the resource cache
-// it does not initialize any client call
-func (wrc *Register) GetKubePolicyDeployment() (*appsv1.Deployment, error) {
- deploy, err := wrc.kDeplLister.Deployments(config.KyvernoNamespace()).Get(config.KyvernoDeploymentName())
- if err != nil {
- return nil, err
- }
- return deploy, nil
-}
-
-func (wrc *Register) constructOwner() metav1.OwnerReference {
- logger := wrc.log
- kubeClusterRoleName, err := wrc.GetKubePolicyClusterRoleName()
- if err != nil {
- logger.Error(err, "failed to get cluster role")
- return metav1.OwnerReference{}
- }
- return metav1.OwnerReference{
- APIVersion: config.ClusterRoleAPIVersion,
- Kind: config.ClusterRoleKind,
- Name: kubeClusterRoleName.GetName(),
- UID: kubeClusterRoleName.GetUID(),
- }
-}
-
-// webhook utils
-
-func generateRules(rule admissionregistrationv1.Rule, operationTypes []admissionregistrationv1.OperationType) []admissionregistrationv1.RuleWithOperations {
- if !reflect.DeepEqual(rule, admissionregistrationv1.Rule{}) {
- return []admissionregistrationv1.RuleWithOperations{{Operations: operationTypes, Rule: rule}}
- }
- return nil
-}
-
-func generateDebugMutatingWebhook(name, url string, caData []byte, timeoutSeconds int32, rule admissionregistrationv1.Rule, operationTypes []admissionregistrationv1.OperationType, failurePolicy admissionregistrationv1.FailurePolicyType) admissionregistrationv1.MutatingWebhook {
- return admissionregistrationv1.MutatingWebhook{
- ReinvocationPolicy: &never,
- Name: name,
- ClientConfig: admissionregistrationv1.WebhookClientConfig{
- URL: &url,
- CABundle: caData,
- },
- SideEffects: &noneOnDryRun,
- AdmissionReviewVersions: []string{"v1beta1"},
- TimeoutSeconds: &timeoutSeconds,
- FailurePolicy: &failurePolicy,
- Rules: generateRules(rule, operationTypes),
- }
-}
-
-func generateDebugValidatingWebhook(name, url string, caData []byte, timeoutSeconds int32, rule admissionregistrationv1.Rule, operationTypes []admissionregistrationv1.OperationType, failurePolicy admissionregistrationv1.FailurePolicyType) admissionregistrationv1.ValidatingWebhook {
- return admissionregistrationv1.ValidatingWebhook{
- Name: name,
- ClientConfig: admissionregistrationv1.WebhookClientConfig{
- URL: &url,
- CABundle: caData,
- },
- SideEffects: &noneOnDryRun,
- AdmissionReviewVersions: []string{"v1beta1"},
- TimeoutSeconds: &timeoutSeconds,
- FailurePolicy: &failurePolicy,
- Rules: generateRules(rule, operationTypes),
- }
-}
-
-func generateMutatingWebhook(name, servicePath string, caData []byte, timeoutSeconds int32, rule admissionregistrationv1.Rule, operationTypes []admissionregistrationv1.OperationType, failurePolicy admissionregistrationv1.FailurePolicyType) admissionregistrationv1.MutatingWebhook {
- return admissionregistrationv1.MutatingWebhook{
- ReinvocationPolicy: &ifNeeded,
- Name: name,
- ClientConfig: admissionregistrationv1.WebhookClientConfig{
- Service: &admissionregistrationv1.ServiceReference{
- Namespace: config.KyvernoNamespace(),
- Name: config.KyvernoServiceName(),
- Path: &servicePath,
- },
- CABundle: caData,
- },
- SideEffects: &noneOnDryRun,
- AdmissionReviewVersions: []string{"v1beta1"},
- TimeoutSeconds: &timeoutSeconds,
- FailurePolicy: &failurePolicy,
- Rules: generateRules(rule, operationTypes),
- }
-}
-
-func generateValidatingWebhook(name, servicePath string, caData []byte, timeoutSeconds int32, rule admissionregistrationv1.Rule, operationTypes []admissionregistrationv1.OperationType, failurePolicy admissionregistrationv1.FailurePolicyType) admissionregistrationv1.ValidatingWebhook {
- return admissionregistrationv1.ValidatingWebhook{
- Name: name,
- ClientConfig: admissionregistrationv1.WebhookClientConfig{
- Service: &admissionregistrationv1.ServiceReference{
- Namespace: config.KyvernoNamespace(),
- Name: config.KyvernoServiceName(),
- Path: &servicePath,
- },
- CABundle: caData,
- },
- SideEffects: &noneOnDryRun,
- AdmissionReviewVersions: []string{"v1beta1"},
- TimeoutSeconds: &timeoutSeconds,
- FailurePolicy: &failurePolicy,
- Rules: generateRules(rule, operationTypes),
- }
-}
-
-func generateObjectMeta(name string, owner ...metav1.OwnerReference) metav1.ObjectMeta {
- return metav1.ObjectMeta{
- Name: name,
- Labels: map[string]string{
- managedByLabel: kyvernov1.ValueKyvernoApp,
- },
- OwnerReferences: owner,
- }
-}
-
-// policy webhook configuration utils
-
-func getPolicyMutatingWebhookConfigName(serverIP string) string {
- if serverIP != "" {
- return config.PolicyMutatingWebhookConfigurationDebugName
- }
- return config.PolicyMutatingWebhookConfigurationName
-}
-
-func getPolicyValidatingWebhookConfigName(serverIP string) string {
- if serverIP != "" {
- return config.PolicyValidatingWebhookConfigurationDebugName
- }
- return config.PolicyValidatingWebhookConfigurationName
-}
-
-func constructPolicyValidatingWebhookConfig(caData []byte, timeoutSeconds int32, owner metav1.OwnerReference) *admissionregistrationv1.ValidatingWebhookConfiguration {
- name, path := config.PolicyValidatingWebhookName, config.PolicyValidatingWebhookServicePath
- return &admissionregistrationv1.ValidatingWebhookConfiguration{
- ObjectMeta: generateObjectMeta(config.PolicyValidatingWebhookConfigurationName, owner),
- Webhooks: []admissionregistrationv1.ValidatingWebhook{
- generateValidatingWebhook(name, path, caData, timeoutSeconds, policyRule, createUpdate, admissionregistrationv1.Ignore),
- },
- }
-}
-
-func constructDebugPolicyValidatingWebhookConfig(serverIP string, caData []byte, timeoutSeconds int32, owner metav1.OwnerReference) *admissionregistrationv1.ValidatingWebhookConfiguration {
- name, url := config.PolicyValidatingWebhookName, fmt.Sprintf("https://%s%s", serverIP, config.PolicyValidatingWebhookServicePath)
- return &admissionregistrationv1.ValidatingWebhookConfiguration{
- ObjectMeta: generateObjectMeta(config.PolicyValidatingWebhookConfigurationDebugName, owner),
- Webhooks: []admissionregistrationv1.ValidatingWebhook{
- generateDebugValidatingWebhook(name, url, caData, timeoutSeconds, policyRule, createUpdate, admissionregistrationv1.Ignore),
- },
- }
-}
-
-func constructPolicyMutatingWebhookConfig(caData []byte, timeoutSeconds int32, owner metav1.OwnerReference) *admissionregistrationv1.MutatingWebhookConfiguration {
- name, path := config.PolicyMutatingWebhookName, config.PolicyMutatingWebhookServicePath
- return &admissionregistrationv1.MutatingWebhookConfiguration{
- ObjectMeta: generateObjectMeta(config.PolicyMutatingWebhookConfigurationName, owner),
- Webhooks: []admissionregistrationv1.MutatingWebhook{
- generateMutatingWebhook(name, path, caData, timeoutSeconds, policyRule, createUpdate, admissionregistrationv1.Ignore),
- },
- }
-}
-
-func constructDebugPolicyMutatingWebhookConfig(serverIP string, caData []byte, timeoutSeconds int32, owner metav1.OwnerReference) *admissionregistrationv1.MutatingWebhookConfiguration {
- name, url := config.PolicyMutatingWebhookName, fmt.Sprintf("https://%s%s", serverIP, config.PolicyMutatingWebhookServicePath)
- return &admissionregistrationv1.MutatingWebhookConfiguration{
- ObjectMeta: generateObjectMeta(config.PolicyMutatingWebhookConfigurationDebugName, owner),
- Webhooks: []admissionregistrationv1.MutatingWebhook{
- generateDebugMutatingWebhook(name, url, caData, timeoutSeconds, policyRule, createUpdate, admissionregistrationv1.Ignore),
- },
- }
-}
-
-// resource webhook configuration utils
-
-func getResourceMutatingWebhookConfigName(serverIP string) string {
- if serverIP != "" {
- return config.MutatingWebhookConfigurationDebugName
- }
- return config.MutatingWebhookConfigurationName
-}
-
-func getResourceValidatingWebhookConfigName(serverIP string) string {
- if serverIP != "" {
- return config.ValidatingWebhookConfigurationDebugName
- }
- return config.ValidatingWebhookConfigurationName
-}
-
-func defaultResourceWebhookRule(autoUpdate bool) admissionregistrationv1.Rule {
- if autoUpdate {
- return admissionregistrationv1.Rule{}
- }
- return admissionregistrationv1.Rule{
- APIGroups: []string{"*"},
- APIVersions: []string{"*"},
- Resources: []string{"*/*"},
- }
-}
-
-func constructDefaultDebugMutatingWebhookConfig(serverIP string, caData []byte, timeoutSeconds int32, autoUpdate bool, owner metav1.OwnerReference) *admissionregistrationv1.MutatingWebhookConfiguration {
- name, baseUrl := config.MutatingWebhookName, fmt.Sprintf("https://%s%s", serverIP, config.MutatingWebhookServicePath)
- url := fmt.Sprintf("%s/ignore", baseUrl)
- webhook := &admissionregistrationv1.MutatingWebhookConfiguration{
- ObjectMeta: generateObjectMeta(config.MutatingWebhookConfigurationDebugName, owner),
- Webhooks: []admissionregistrationv1.MutatingWebhook{
- generateDebugMutatingWebhook(name+"-ignore", url, caData, timeoutSeconds, defaultResourceWebhookRule(autoUpdate), createUpdate, admissionregistrationv1.Ignore),
- },
- }
- if autoUpdate {
- url := fmt.Sprintf("%s/fail", baseUrl)
- webhook.Webhooks = append(webhook.Webhooks, generateDebugMutatingWebhook(name+"-fail", url, caData, timeoutSeconds, defaultResourceWebhookRule(autoUpdate), createUpdate, admissionregistrationv1.Fail))
- }
- return webhook
-}
-
-func constructDefaultMutatingWebhookConfig(caData []byte, timeoutSeconds int32, autoUpdate bool, owner metav1.OwnerReference) *admissionregistrationv1.MutatingWebhookConfiguration {
- name, basePath := config.MutatingWebhookName, config.MutatingWebhookServicePath
- path := fmt.Sprintf("%s/ignore", basePath)
- webhook := &admissionregistrationv1.MutatingWebhookConfiguration{
- ObjectMeta: generateObjectMeta(config.MutatingWebhookConfigurationName, owner),
- Webhooks: []admissionregistrationv1.MutatingWebhook{
- generateMutatingWebhook(name+"-ignore", path, caData, timeoutSeconds, defaultResourceWebhookRule(autoUpdate), createUpdate, admissionregistrationv1.Ignore),
- },
- }
- if autoUpdate {
- path := fmt.Sprintf("%s/fail", basePath)
- webhook.Webhooks = append(webhook.Webhooks, generateMutatingWebhook(name+"-fail", path, caData, timeoutSeconds, defaultResourceWebhookRule(autoUpdate), createUpdate, admissionregistrationv1.Fail))
- }
- return webhook
-}
-
-func constructDefaultDebugValidatingWebhookConfig(serverIP string, caData []byte, timeoutSeconds int32, autoUpdate bool, owner metav1.OwnerReference) *admissionregistrationv1.ValidatingWebhookConfiguration {
- name, baseUrl := config.ValidatingWebhookName, fmt.Sprintf("https://%s%s", serverIP, config.ValidatingWebhookServicePath)
- url := fmt.Sprintf("%s/ignore", baseUrl)
- webhook := &admissionregistrationv1.ValidatingWebhookConfiguration{
- ObjectMeta: generateObjectMeta(config.ValidatingWebhookConfigurationDebugName, owner),
- Webhooks: []admissionregistrationv1.ValidatingWebhook{
- generateDebugValidatingWebhook(name+"-ignore", url, caData, timeoutSeconds, defaultResourceWebhookRule(autoUpdate), all, admissionregistrationv1.Ignore),
- },
- }
- if autoUpdate {
- url := fmt.Sprintf("%s/fail", baseUrl)
- webhook.Webhooks = append(webhook.Webhooks, generateDebugValidatingWebhook(name+"-fail", url, caData, timeoutSeconds, defaultResourceWebhookRule(autoUpdate), all, admissionregistrationv1.Fail))
- }
- return webhook
-}
-
-func constructDefaultValidatingWebhookConfig(caData []byte, timeoutSeconds int32, autoUpdate bool, owner metav1.OwnerReference) *admissionregistrationv1.ValidatingWebhookConfiguration {
- name, basePath := config.ValidatingWebhookName, config.ValidatingWebhookServicePath
- path := fmt.Sprintf("%s/ignore", basePath)
- webhook := &admissionregistrationv1.ValidatingWebhookConfiguration{
- ObjectMeta: generateObjectMeta(config.ValidatingWebhookConfigurationName, owner),
- Webhooks: []admissionregistrationv1.ValidatingWebhook{
- generateValidatingWebhook(name+"-ignore", path, caData, timeoutSeconds, defaultResourceWebhookRule(autoUpdate), all, admissionregistrationv1.Ignore),
- },
- }
- if autoUpdate {
- path := fmt.Sprintf("%s/fail", basePath)
- webhook.Webhooks = append(webhook.Webhooks, generateValidatingWebhook(name+"-fail", path, caData, timeoutSeconds, defaultResourceWebhookRule(autoUpdate), all, admissionregistrationv1.Fail))
- }
- return webhook
-}
-
-// verify webhook configuration utils
-
-func getVerifyMutatingWebhookConfigName(serverIP string) string {
- if serverIP != "" {
- return config.VerifyMutatingWebhookConfigurationDebugName
- }
- return config.VerifyMutatingWebhookConfigurationName
-}
-
-func constructVerifyMutatingWebhookConfig(caData []byte, timeoutSeconds int32, owner metav1.OwnerReference) *admissionregistrationv1.MutatingWebhookConfiguration {
- name, path := config.VerifyMutatingWebhookName, config.VerifyMutatingWebhookServicePath
- webhook := generateMutatingWebhook(name, path, caData, timeoutSeconds, verifyRule, update, admissionregistrationv1.Ignore)
- webhook.ObjectSelector = vertifyObjectSelector
- return &admissionregistrationv1.MutatingWebhookConfiguration{
- ObjectMeta: generateObjectMeta(config.VerifyMutatingWebhookConfigurationName, owner),
- Webhooks: []admissionregistrationv1.MutatingWebhook{webhook},
- }
-}
-
-func constructDebugVerifyMutatingWebhookConfig(serverIP string, caData []byte, timeoutSeconds int32, owner metav1.OwnerReference) *admissionregistrationv1.MutatingWebhookConfiguration {
- name, url := config.VerifyMutatingWebhookName, fmt.Sprintf("https://%s%s", serverIP, config.VerifyMutatingWebhookServicePath)
- webhook := generateDebugMutatingWebhook(name, url, caData, timeoutSeconds, verifyRule, update, admissionregistrationv1.Ignore)
- webhook.ObjectSelector = vertifyObjectSelector
- return &admissionregistrationv1.MutatingWebhookConfiguration{
- ObjectMeta: generateObjectMeta(config.VerifyMutatingWebhookConfigurationDebugName, owner),
- Webhooks: []admissionregistrationv1.MutatingWebhook{webhook},
- }
-}
diff --git a/pkg/webhookconfig/configmanager.go b/pkg/webhookconfig/configmanager.go
deleted file mode 100644
index 470257b88b..0000000000
--- a/pkg/webhookconfig/configmanager.go
+++ /dev/null
@@ -1,681 +0,0 @@
-package webhookconfig
-
-import (
- "context"
- "reflect"
- "strings"
- "sync/atomic"
- "time"
-
- "github.com/go-logr/logr"
- kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
- "github.com/kyverno/kyverno/pkg/autogen"
- "github.com/kyverno/kyverno/pkg/client/clientset/versioned"
- kyvernov1informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1"
- kyvernov1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1"
- "github.com/kyverno/kyverno/pkg/clients/dclient"
- "github.com/kyverno/kyverno/pkg/config"
- "github.com/kyverno/kyverno/pkg/metrics"
- "github.com/kyverno/kyverno/pkg/toggle"
- "github.com/kyverno/kyverno/pkg/utils"
- kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
- "github.com/pkg/errors"
- admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
- apierrors "k8s.io/apimachinery/pkg/api/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/labels"
- "k8s.io/apimachinery/pkg/runtime/schema"
- utilruntime "k8s.io/apimachinery/pkg/util/runtime"
- "k8s.io/apimachinery/pkg/util/sets"
- admissionregistrationv1informers "k8s.io/client-go/informers/admissionregistration/v1"
- "k8s.io/client-go/kubernetes"
- admissionregistrationv1listers "k8s.io/client-go/listers/admissionregistration/v1"
- "k8s.io/client-go/tools/cache"
- "k8s.io/client-go/util/workqueue"
-)
-
-var DefaultWebhookTimeout int32 = 10
-
-// webhookConfigManager manges the webhook configuration dynamically
-// it is NOT multi-thread safe
-type webhookConfigManager struct {
- // clients
- discoveryClient dclient.IDiscovery
- kubeClient kubernetes.Interface
- kyvernoClient versioned.Interface
-
- // informers
- pInformer kyvernov1informers.ClusterPolicyInformer
- npInformer kyvernov1informers.PolicyInformer
- mutateInformer admissionregistrationv1informers.MutatingWebhookConfigurationInformer
- validateInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer
-
- // listers
- pLister kyvernov1listers.ClusterPolicyLister
- npLister kyvernov1listers.PolicyLister
- mutateLister admissionregistrationv1listers.MutatingWebhookConfigurationLister
- validateLister admissionregistrationv1listers.ValidatingWebhookConfigurationLister
-
- // queue
- queue workqueue.RateLimitingInterface
-
- metricsConfig metrics.MetricsConfigManager
-
- // serverIP used to get the name of debug webhooks
- serverIP string
- autoUpdateWebhooks bool
-
- // wildcardPolicy indicates the number of policies that matches all kinds (*) defined
- wildcardPolicy int64
-
- createDefaultWebhook chan<- string
-
- stopCh <-chan struct{}
-
- log logr.Logger
-}
-
-type manage interface {
- start()
-}
-
-func newWebhookConfigManager(
- ctx context.Context,
- discoveryClient dclient.IDiscovery,
- kubeClient kubernetes.Interface,
- kyvernoClient versioned.Interface,
- pInformer kyvernov1informers.ClusterPolicyInformer,
- npInformer kyvernov1informers.PolicyInformer,
- mwcInformer admissionregistrationv1informers.MutatingWebhookConfigurationInformer,
- vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer,
- metricsConfig metrics.MetricsConfigManager,
- serverIP string,
- autoUpdateWebhooks bool,
- createDefaultWebhook chan<- string,
- log logr.Logger,
-) manage {
- m := &webhookConfigManager{
- discoveryClient: discoveryClient,
- kyvernoClient: kyvernoClient,
- kubeClient: kubeClient,
- pInformer: pInformer,
- npInformer: npInformer,
- mutateInformer: mwcInformer,
- validateInformer: vwcInformer,
- pLister: pInformer.Lister(),
- npLister: npInformer.Lister(),
- mutateLister: mwcInformer.Lister(),
- validateLister: vwcInformer.Lister(),
- metricsConfig: metricsConfig,
- queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "configmanager"),
- wildcardPolicy: 0,
- serverIP: serverIP,
- autoUpdateWebhooks: autoUpdateWebhooks,
- createDefaultWebhook: createDefaultWebhook,
- stopCh: ctx.Done(),
- log: log,
- }
-
- return m
-}
-
-func (m *webhookConfigManager) handleErr(err error, key interface{}) {
- logger := m.log
- if err == nil {
- m.queue.Forget(key)
- return
- }
- if m.queue.NumRequeues(key) < 3 {
- logger.Error(err, "failed to sync policy", "key", key)
- m.queue.AddRateLimited(key)
- return
- }
- utilruntime.HandleError(err)
- logger.V(2).Info("dropping policy out of queue", "key", key)
- m.queue.Forget(key)
-}
-
-func (m *webhookConfigManager) addClusterPolicy(obj interface{}) {
- p := obj.(*kyvernov1.ClusterPolicy)
- if hasWildcard(&p.Spec) {
- atomic.AddInt64(&m.wildcardPolicy, int64(1))
- }
- m.enqueue(p)
-}
-
-func (m *webhookConfigManager) updateClusterPolicy(old, cur interface{}) {
- oldP, curP := old.(*kyvernov1.ClusterPolicy), cur.(*kyvernov1.ClusterPolicy)
- if reflect.DeepEqual(oldP.Spec, curP.Spec) {
- return
- }
- if hasWildcard(&oldP.Spec) && !hasWildcard(&curP.Spec) {
- atomic.AddInt64(&m.wildcardPolicy, ^int64(0))
- } else if !hasWildcard(&oldP.Spec) && hasWildcard(&curP.Spec) {
- atomic.AddInt64(&m.wildcardPolicy, int64(1))
- }
- m.enqueue(curP)
-}
-
-func (m *webhookConfigManager) deleteClusterPolicy(obj interface{}) {
- p, ok := kubeutils.GetObjectWithTombstone(obj).(*kyvernov1.ClusterPolicy)
- if !ok {
- // utilruntime.HandleError(fmt.Errorf("error decoding object tombstone, invalid type"))
- m.log.V(2).Info("Failed to get deleted object", "obj", obj)
- return
- }
- if hasWildcard(&p.Spec) {
- atomic.AddInt64(&m.wildcardPolicy, ^int64(0))
- }
- m.enqueue(p)
-}
-
-func (m *webhookConfigManager) addPolicy(obj interface{}) {
- p := obj.(*kyvernov1.Policy)
- if hasWildcard(&p.Spec) {
- atomic.AddInt64(&m.wildcardPolicy, int64(1))
- }
- m.enqueue(p)
-}
-
-func (m *webhookConfigManager) updatePolicy(old, cur interface{}) {
- oldP, curP := old.(*kyvernov1.Policy), cur.(*kyvernov1.Policy)
- if reflect.DeepEqual(oldP.Spec, curP.Spec) {
- return
- }
- if hasWildcard(&oldP.Spec) && !hasWildcard(&curP.Spec) {
- atomic.AddInt64(&m.wildcardPolicy, ^int64(0))
- } else if !hasWildcard(&oldP.Spec) && hasWildcard(&curP.Spec) {
- atomic.AddInt64(&m.wildcardPolicy, int64(1))
- }
- m.enqueue(curP)
-}
-
-func (m *webhookConfigManager) deletePolicy(obj interface{}) {
- p, ok := kubeutils.GetObjectWithTombstone(obj).(*kyvernov1.Policy)
- if !ok {
- // utilruntime.HandleError(fmt.Errorf("error decoding object tombstone, invalid type"))
- m.log.V(2).Info("Failed to get deleted object", "obj", obj)
- return
- }
- if hasWildcard(&p.Spec) {
- atomic.AddInt64(&m.wildcardPolicy, ^int64(0))
- }
- m.enqueue(p)
-}
-
-func (m *webhookConfigManager) deleteMutatingWebhook(obj interface{}) {
- m.log.WithName("deleteMutatingWebhook").Info("resource webhook configuration was deleted, recreating...")
- webhook, ok := kubeutils.GetObjectWithTombstone(obj).(*admissionregistrationv1.MutatingWebhookConfiguration)
- if !ok {
- m.log.V(2).Info("Failed to get deleted object", "obj", obj)
- return
- }
- if webhook.GetName() == config.MutatingWebhookConfigurationName {
- m.enqueueAllPolicies()
- }
-}
-
-func (m *webhookConfigManager) deleteValidatingWebhook(obj interface{}) {
- m.log.WithName("deleteMutatingWebhook").Info("resource webhook configuration was deleted, recreating...")
- webhook, ok := kubeutils.GetObjectWithTombstone(obj).(*admissionregistrationv1.ValidatingWebhookConfiguration)
- if !ok {
- m.log.V(2).Info("Failed to get deleted object", "obj", obj)
- return
- }
- if webhook.GetName() == config.ValidatingWebhookConfigurationName {
- m.enqueueAllPolicies()
- }
-}
-
-func (m *webhookConfigManager) enqueueAllPolicies() {
- logger := m.log.WithName("enqueueAllPolicies")
- policies, err := m.listAllPolicies()
- if err != nil {
- logger.Error(err, "unable to list policies")
- }
- for _, policy := range policies {
- m.enqueue(policy)
- logger.V(4).Info("added policy to the queue", "namespace", policy.GetNamespace(), "name", policy.GetName())
- }
-}
-
-func (m *webhookConfigManager) enqueue(policy interface{}) {
- logger := m.log
- key, err := cache.MetaNamespaceKeyFunc(policy)
- if err != nil {
- logger.Error(err, "failed to enqueue policy")
- return
- }
- m.queue.Add(key)
-}
-
-// start is a blocking call to configure webhook
-func (m *webhookConfigManager) start() {
- defer utilruntime.HandleCrash()
- defer m.queue.ShutDown()
-
- m.log.V(2).Info("starting")
- defer m.log.V(2).Info("shutting down")
-
- m.pInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
- AddFunc: m.addClusterPolicy,
- UpdateFunc: m.updateClusterPolicy,
- DeleteFunc: m.deleteClusterPolicy,
- })
-
- m.npInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
- AddFunc: m.addPolicy,
- UpdateFunc: m.updatePolicy,
- DeleteFunc: m.deletePolicy,
- })
-
- m.mutateInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
- DeleteFunc: m.deleteMutatingWebhook,
- })
-
- m.validateInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
- DeleteFunc: m.deleteValidatingWebhook,
- })
-
- for m.processNextWorkItem() {
- }
-}
-
-func (m *webhookConfigManager) processNextWorkItem() bool {
- key, quit := m.queue.Get()
- if quit {
- return false
- }
- defer m.queue.Done(key)
- err := m.sync(key.(string))
- m.handleErr(err, key)
- return true
-}
-
-func (m *webhookConfigManager) sync(key string) error {
- logger := m.log.WithName("sync")
- startTime := time.Now()
- logger.V(4).Info("started syncing policy", "key", key, "startTime", startTime)
- defer func() {
- logger.V(4).Info("finished syncing policy", "key", key, "processingTime", time.Since(startTime).String())
- }()
- namespace, name, err := cache.SplitMetaNamespaceKey(key)
- if err != nil {
- logger.Info("invalid resource key", "key", key)
- return nil
- }
- return m.reconcileWebhook(namespace, name)
-}
-
-func (m *webhookConfigManager) reconcileWebhook(namespace, name string) error {
- logger := m.log.WithName("reconcileWebhook").WithValues("namespace", namespace, "policy", name)
-
- _, err := m.getPolicy(namespace, name)
- isDeleted := apierrors.IsNotFound(err)
- if err != nil && !isDeleted {
- return errors.Wrapf(err, "unable to get policy object %s/%s", namespace, name)
- }
-
- ready := true
- var updateErr error
- // build webhook only if auto-update is enabled, otherwise directly update status to ready
- if m.autoUpdateWebhooks {
- webhooks, err := m.buildWebhooks(namespace)
- if err != nil {
- return err
- }
-
- if err := m.updateWebhookConfig(webhooks); err != nil {
- ready = false
- updateErr = errors.Wrapf(err, "failed to update webhook configurations for policy")
- }
-
- // DELETION of the policy
- if isDeleted {
- return nil
- }
- }
-
- if err := m.updateStatus(namespace, name, ready); err != nil {
- return errors.Wrapf(err, "failed to update policy status %s/%s", namespace, name)
- }
-
- if ready {
- logger.Info("policy is ready to serve admission requests")
- }
- return updateErr
-}
-
-func (m *webhookConfigManager) getPolicy(namespace, name string) (kyvernov1.PolicyInterface, error) {
- if namespace == "" {
- return m.pLister.Get(name)
- } else {
- return m.npLister.Policies(namespace).Get(name)
- }
-}
-
-func (m *webhookConfigManager) listAllPolicies() ([]kyvernov1.PolicyInterface, error) {
- policies := []kyvernov1.PolicyInterface{}
- polList, err := m.npLister.Policies(metav1.NamespaceAll).List(labels.Everything())
- if err != nil {
- return nil, errors.Wrapf(err, "failed to list Policy")
- }
- for _, p := range polList {
- policies = append(policies, p)
- }
- cpolList, err := m.pLister.List(labels.Everything())
- if err != nil {
- return nil, errors.Wrapf(err, "failed to list ClusterPolicy")
- }
- for _, p := range cpolList {
- policies = append(policies, p)
- }
- return policies, nil
-}
-
-func (m *webhookConfigManager) buildWebhooks(namespace string) (res []*webhook, err error) {
- mutateIgnore := newWebhook(kindMutating, DefaultWebhookTimeout, kyvernov1.Ignore)
- mutateFail := newWebhook(kindMutating, DefaultWebhookTimeout, kyvernov1.Fail)
- validateIgnore := newWebhook(kindValidating, DefaultWebhookTimeout, kyvernov1.Ignore)
- validateFail := newWebhook(kindValidating, DefaultWebhookTimeout, kyvernov1.Fail)
-
- if atomic.LoadInt64(&m.wildcardPolicy) != 0 {
- for _, w := range []*webhook{mutateIgnore, mutateFail, validateIgnore, validateFail} {
- setWildcardConfig(w)
- }
-
- m.log.V(4).WithName("buildWebhooks").Info("warning: found wildcard policy, setting webhook configurations to accept admission requests of all kinds")
- return append(res, mutateIgnore, mutateFail, validateIgnore, validateFail), nil
- }
-
- policies, err := m.listAllPolicies()
- if err != nil {
- return nil, errors.Wrap(err, "unable to list current policies")
- }
-
- for _, p := range policies {
- spec := p.GetSpec()
- if spec.HasValidate() || spec.HasGenerate() || spec.HasMutate() || spec.HasImagesValidationChecks() || spec.HasYAMLSignatureVerify() {
- if spec.GetFailurePolicy() == kyvernov1.Ignore {
- m.mergeWebhook(validateIgnore, p, true)
- } else {
- m.mergeWebhook(validateFail, p, true)
- }
- }
-
- if spec.HasMutate() || spec.HasVerifyImages() {
- if spec.GetFailurePolicy() == kyvernov1.Ignore {
- m.mergeWebhook(mutateIgnore, p, false)
- } else {
- m.mergeWebhook(mutateFail, p, false)
- }
- }
- }
-
- res = append(res, mutateIgnore, mutateFail, validateIgnore, validateFail)
- return res, nil
-}
-
-func (m *webhookConfigManager) updateWebhookConfig(webhooks []*webhook) error {
- logger := m.log.WithName("updateWebhookConfig")
-
- webhooksMap := map[string]*webhook{}
- for _, w := range webhooks {
- webhooksMap[webhookKey(w.kind, string(w.failurePolicy))] = w
- }
-
- var errs []string
- if err := m.updateMutatingWebhookConfiguration(getResourceMutatingWebhookConfigName(m.serverIP), webhooksMap); err != nil {
- logger.V(4).Info("failed to update mutatingwebhookconfigurations", "error", err.Error())
- errs = append(errs, err.Error())
- }
-
- if err := m.updateValidatingWebhookConfiguration(getResourceValidatingWebhookConfigName(m.serverIP), webhooksMap); err != nil {
- logger.V(4).Info("failed to update validatingwebhookconfigurations", "error", err.Error())
- errs = append(errs, err.Error())
- }
-
- if len(errs) != 0 {
- return errors.New(strings.Join(errs, "\n"))
- }
-
- return nil
-}
-
-func (m *webhookConfigManager) updateMutatingWebhookConfiguration(webhookName string, webhooksMap map[string]*webhook) error {
- logger := m.log.WithName("updateMutatingWebhookConfiduration").WithValues("name", webhookName)
- resourceWebhook, err := m.mutateLister.Get(webhookName)
- if err != nil && !apierrors.IsNotFound(err) {
- return errors.Wrapf(err, "unable to get %s/%s", kindMutating, webhookName)
- } else if apierrors.IsNotFound(err) {
- m.createDefaultWebhook <- kindMutating
- return err
- }
- for i := range resourceWebhook.Webhooks {
- newWebhook := webhooksMap[webhookKey(kindMutating, string(*resourceWebhook.Webhooks[i].FailurePolicy))]
- if newWebhook == nil || newWebhook.isEmpty() {
- resourceWebhook.Webhooks[i].Rules = []admissionregistrationv1.RuleWithOperations{}
- } else {
- resourceWebhook.Webhooks[i].TimeoutSeconds = &newWebhook.maxWebhookTimeout
- resourceWebhook.Webhooks[i].Rules = []admissionregistrationv1.RuleWithOperations{
- newWebhook.buildRuleWithOperations(admissionregistrationv1.Create, admissionregistrationv1.Update, admissionregistrationv1.Delete),
- }
- }
- }
- if _, err := m.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.TODO(), resourceWebhook, metav1.UpdateOptions{}); err != nil {
- m.metricsConfig.RecordClientQueries(metrics.ClientUpdate, metrics.KubeClient, kindMutating, "")
- return errors.Wrapf(err, "unable to update: %s", resourceWebhook.GetName())
- }
- logger.V(4).Info("successfully updated the webhook configuration")
- return nil
-}
-
-func (m *webhookConfigManager) updateValidatingWebhookConfiguration(webhookName string, webhooksMap map[string]*webhook) error {
- logger := m.log.WithName("updateMutatingWebhookConfiduration").WithValues("name", webhookName)
- resourceWebhook, err := m.validateLister.Get(webhookName)
- if err != nil && !apierrors.IsNotFound(err) {
- return errors.Wrapf(err, "unable to get %s/%s", kindValidating, webhookName)
- } else if apierrors.IsNotFound(err) {
- m.createDefaultWebhook <- kindValidating
- return err
- }
- for i := range resourceWebhook.Webhooks {
- newWebhook := webhooksMap[webhookKey(kindValidating, string(*resourceWebhook.Webhooks[i].FailurePolicy))]
- if newWebhook == nil || newWebhook.isEmpty() {
- resourceWebhook.Webhooks[i].Rules = []admissionregistrationv1.RuleWithOperations{}
- } else {
- resourceWebhook.Webhooks[i].TimeoutSeconds = &newWebhook.maxWebhookTimeout
- resourceWebhook.Webhooks[i].Rules = []admissionregistrationv1.RuleWithOperations{
- newWebhook.buildRuleWithOperations(admissionregistrationv1.Create, admissionregistrationv1.Update, admissionregistrationv1.Delete, admissionregistrationv1.Connect),
- }
- }
- }
- if _, err := m.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(context.TODO(), resourceWebhook, metav1.UpdateOptions{}); err != nil {
- m.metricsConfig.RecordClientQueries(metrics.ClientUpdate, metrics.KubeClient, kindValidating, "")
- return errors.Wrapf(err, "unable to update: %s", resourceWebhook.GetName())
- }
- logger.V(4).Info("successfully updated the webhook configuration")
- return nil
-}
-
-func (m *webhookConfigManager) updateStatus(namespace, name string, ready bool) error {
- update := func(meta *metav1.ObjectMeta, p kyvernov1.PolicyInterface, status *kyvernov1.PolicyStatus) bool {
- copy := status.DeepCopy()
- status.SetReady(ready)
- if toggle.AutogenInternals.Enabled() {
- var rules []kyvernov1.Rule
- for _, rule := range autogen.ComputeRules(p) {
- if strings.HasPrefix(rule.Name, "autogen-") {
- rules = append(rules, rule)
- }
- }
- status.Autogen.Rules = rules
- } else {
- status.Autogen.Rules = nil
- }
- return !reflect.DeepEqual(status, copy)
- }
- if namespace == "" {
- p, err := m.pLister.Get(name)
- if err != nil {
- return err
- }
- if update(&p.ObjectMeta, p, &p.Status) {
- if _, err := m.kyvernoClient.KyvernoV1().ClusterPolicies().UpdateStatus(context.TODO(), p, metav1.UpdateOptions{}); err != nil {
- return err
- }
- }
- } else {
- p, err := m.npLister.Policies(namespace).Get(name)
- if err != nil {
- return err
- }
- if update(&p.ObjectMeta, p, &p.Status) {
- if _, err := m.kyvernoClient.KyvernoV1().Policies(namespace).UpdateStatus(context.TODO(), p, metav1.UpdateOptions{}); err != nil {
- return err
- }
- }
- }
- return nil
-}
-
-// webhook is the instance that aggregates the GVK of existing policies
-// based on kind, failurePolicy and webhookTimeout
-type webhook struct {
- kind string
- maxWebhookTimeout int32
- failurePolicy kyvernov1.FailurePolicyType
- groups sets.String
- versions sets.String
- resources sets.String
-}
-
-func (wh *webhook) buildRuleWithOperations(ops ...admissionregistrationv1.OperationType) admissionregistrationv1.RuleWithOperations {
- return admissionregistrationv1.RuleWithOperations{
- Rule: admissionregistrationv1.Rule{
- APIGroups: wh.groups.List(),
- APIVersions: wh.versions.List(),
- Resources: wh.resources.List(),
- },
- Operations: ops,
- }
-}
-
-func (wh *webhook) isEmpty() bool {
- return wh.groups.Len() == 0 || wh.versions.Len() == 0 || wh.resources.Len() == 0
-}
-
-// mergeWebhook merges the matching kinds of the policy to webhook.rule
-func (m *webhookConfigManager) mergeWebhook(dst *webhook, policy kyvernov1.PolicyInterface, updateValidate bool) {
- matchedGVK := make([]string, 0)
- for _, rule := range autogen.ComputeRules(policy) {
- // matching kinds in generate policies need to be added to both webhook
- if rule.HasGenerate() {
- matchedGVK = append(matchedGVK, rule.MatchResources.GetKinds()...)
- matchedGVK = append(matchedGVK, rule.Generation.ResourceSpec.Kind)
- continue
- }
-
- if (updateValidate && rule.HasValidate() || rule.HasImagesValidationChecks()) ||
- (updateValidate && rule.HasMutate() && rule.IsMutateExisting()) ||
- (!updateValidate && rule.HasMutate()) && !rule.IsMutateExisting() ||
- (!updateValidate && rule.HasVerifyImages()) || (!updateValidate && rule.HasYAMLSignatureVerify()) {
- matchedGVK = append(matchedGVK, rule.MatchResources.GetKinds()...)
- }
- }
-
- gvkMap := make(map[string]int)
- gvrList := make([]schema.GroupVersionResource, 0)
- for _, gvk := range matchedGVK {
- if _, ok := gvkMap[gvk]; !ok {
- gvkMap[gvk] = 1
-
- // note: webhook stores GVR in its rules while policy stores GVK in its rules definition
- gv, k := kubeutils.GetKindFromGVK(gvk)
- switch k {
- case "Binding":
- gvrList = append(gvrList, schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods/binding"})
- case "NodeProxyOptions":
- gvrList = append(gvrList, schema.GroupVersionResource{Group: "", Version: "v1", Resource: "nodes/proxy"})
- case "PodAttachOptions":
- gvrList = append(gvrList, schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods/attach"})
- case "PodExecOptions":
- gvrList = append(gvrList, schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods/exec"})
- case "PodPortForwardOptions":
- gvrList = append(gvrList, schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods/portforward"})
- case "PodProxyOptions":
- gvrList = append(gvrList, schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods/proxy"})
- case "ServiceProxyOptions":
- gvrList = append(gvrList, schema.GroupVersionResource{Group: "", Version: "v1", Resource: "services/proxy"})
- default:
- _, gvr, err := m.discoveryClient.FindResource(gv, k)
- if err != nil {
- m.log.Error(err, "unable to convert GVK to GVR", "GVK", gvk)
- continue
- }
- if strings.Contains(gvk, "*") {
- group := kubeutils.GetGroupFromGVK(gvk)
- gvrList = append(gvrList, schema.GroupVersionResource{Group: group, Version: "*", Resource: gvr.Resource})
- } else {
- m.log.V(4).Info("configuring webhook", "GVK", gvk, "GVR", gvr)
- gvrList = append(gvrList, gvr)
- }
- }
- }
- }
-
- for _, gvr := range gvrList {
- dst.groups.Insert(gvr.Group)
- if gvr.Version == "*" {
- dst.versions = sets.NewString()
- dst.versions.Insert(gvr.Version)
- } else if !dst.versions.Has("*") {
- dst.versions.Insert(gvr.Version)
- }
- dst.resources.Insert(gvr.Resource)
- }
-
- if dst.resources.Has("pods") {
- dst.resources.Insert("pods/ephemeralcontainers")
- }
- if dst.resources.Has("services") {
- dst.resources.Insert("services/status")
- }
-
- spec := policy.GetSpec()
- if spec.WebhookTimeoutSeconds != nil {
- if dst.maxWebhookTimeout < *spec.WebhookTimeoutSeconds {
- dst.maxWebhookTimeout = *spec.WebhookTimeoutSeconds
- }
- }
-}
-
-func newWebhook(kind string, timeout int32, failurePolicy kyvernov1.FailurePolicyType) *webhook {
- return &webhook{
- kind: kind,
- maxWebhookTimeout: timeout,
- failurePolicy: failurePolicy,
- groups: sets.NewString(),
- versions: sets.NewString(),
- resources: sets.NewString(),
- }
-}
-
-func webhookKey(webhookKind, failurePolicy string) string {
- return strings.Join([]string{webhookKind, failurePolicy}, "/")
-}
-
-func hasWildcard(spec *kyvernov1.Spec) bool {
- for _, rule := range spec.Rules {
- if kinds := rule.MatchResources.GetKinds(); utils.ContainsString(kinds, "*") {
- return true
- }
- }
- return false
-}
-
-func setWildcardConfig(w *webhook) {
- w.groups = sets.NewString("*")
- w.versions = sets.NewString("*")
- w.resources = sets.NewString("*/*")
-}
diff --git a/pkg/webhookconfig/configmanager_test.go b/pkg/webhookconfig/configmanager_test.go
deleted file mode 100644
index ee73fc1295..0000000000
--- a/pkg/webhookconfig/configmanager_test.go
+++ /dev/null
@@ -1,16 +0,0 @@
-package webhookconfig
-
-import (
- "testing"
-
- kyverno "github.com/kyverno/kyverno/api/kyverno/v1"
- "gotest.tools/assert"
-)
-
-func Test_webhook_isEmpty(t *testing.T) {
- empty := newWebhook(kindMutating, DefaultWebhookTimeout, kyverno.Ignore)
- assert.Equal(t, empty.isEmpty(), true)
- notEmpty := newWebhook(kindMutating, DefaultWebhookTimeout, kyverno.Ignore)
- setWildcardConfig(notEmpty)
- assert.Equal(t, notEmpty.isEmpty(), false)
-}
diff --git a/pkg/webhookconfig/monitor.go b/pkg/webhookconfig/monitor.go
deleted file mode 100644
index baf0b50f84..0000000000
--- a/pkg/webhookconfig/monitor.go
+++ /dev/null
@@ -1,235 +0,0 @@
-package webhookconfig
-
-import (
- "context"
- "fmt"
- "sync"
- "time"
-
- "github.com/go-logr/logr"
- "github.com/kyverno/kyverno/pkg/config"
- "github.com/kyverno/kyverno/pkg/event"
- "github.com/kyverno/kyverno/pkg/tls"
- "github.com/pkg/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/client-go/kubernetes"
- coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1"
-)
-
-// maxRetryCount defines the max deadline count
-const (
- tickerInterval time.Duration = 30 * time.Second
- idleCheckInterval time.Duration = 60 * time.Second
- idleDeadline time.Duration = idleCheckInterval * 5
-)
-
-// Monitor stores the last webhook request time and monitors registered webhooks.
-//
-// If a webhook is not received in the idleCheckInterval the monitor triggers a
-// change in the Kyverno deployment to force a webhook request. If no requests
-// are received after idleDeadline the webhooks are deleted and re-registered.
-//
-// Each instance has an in-memory flag lastSeenRequestTime, recording the last
-// received admission timestamp by the current instance. And the latest timestamp
-// (latestTimestamp) is recorded in the annotation of the Kyverno deployment,
-// this annotation could be updated by any instance. If the duration from
-// latestTimestamp is longer than idleCheckInterval, the monitor triggers an
-// annotation update; otherwise lastSeenRequestTime is updated to latestTimestamp.
-//
-// Webhook configurations are checked every tickerInterval across all instances.
-// Currently the check only queries for the expected resource name, and does
-// not compare other details like the webhook settings.
-type Monitor struct {
- // leaseClient is used to manage Kyverno lease
- leaseClient coordinationv1.LeaseInterface
-
- // lastSeenRequestTime records the timestamp
- // of the latest received admission request
- lastSeenRequestTime time.Time
- mu sync.RWMutex
-
- log logr.Logger
-}
-
-// NewMonitor returns a new instance of webhook monitor
-func NewMonitor(kubeClient kubernetes.Interface, log logr.Logger) (*Monitor, error) {
- monitor := &Monitor{
- leaseClient: kubeClient.CoordinationV1().Leases(config.KyvernoNamespace()),
- lastSeenRequestTime: time.Now(),
- log: log,
- }
-
- return monitor, nil
-}
-
-// Time returns the last request time
-func (t *Monitor) Time() time.Time {
- t.mu.RLock()
- defer t.mu.RUnlock()
- return t.lastSeenRequestTime
-}
-
-// SetTime updates the last request time
-func (t *Monitor) SetTime(tm time.Time) {
- t.mu.Lock()
- defer t.mu.Unlock()
-
- t.lastSeenRequestTime = tm
-}
-
-// Run runs the checker and verify the resource update
-func (t *Monitor) Run(ctx context.Context, register *Register, certRenewer *tls.CertRenewer, eventGen event.Interface) {
- logger := t.log.WithName("webhookMonitor")
-
- logger.V(3).Info("starting webhook monitor", "interval", idleCheckInterval.String())
- status := newStatusControl(t.leaseClient, eventGen, logger.WithName("WebhookStatusControl"))
-
- ticker := time.NewTicker(tickerInterval)
- defer ticker.Stop()
-
- createDefaultWebhook := register.createDefaultWebhook
- for {
- select {
- case webhookKind := <-createDefaultWebhook:
- logger.Info("received recreation request for resource webhook")
- if webhookKind == kindMutating {
- err := register.createResourceMutatingWebhookConfiguration(register.readCaData())
- if err != nil {
- logger.Error(err, "failed to create default MutatingWebhookConfiguration for resources, the webhook will be reconciled", "interval", tickerInterval)
- }
- } else if webhookKind == kindValidating {
- err := register.createResourceValidatingWebhookConfiguration(register.readCaData())
- if err != nil {
- logger.Error(err, "failed to create default ValidatingWebhookConfiguration for resources, the webhook will be reconciled", "interval", tickerInterval)
- }
- }
-
- case <-ticker.C:
-
- err := registerWebhookIfNotPresent(register, t.log.WithName("registerWebhookIfNotPresent"))
- if err != nil {
- t.log.Error(err, "")
- }
-
- // update namespaceSelector every 30 seconds
- go func() {
- if register.autoUpdateWebhooks {
- select {
- case register.UpdateWebhookChan <- true:
- logger.V(4).Info("updating webhook configurations for namespaceSelector with latest kyverno ConfigMap")
- default:
- logger.V(4).Info("skipped sending update webhook signal as the channel was blocking")
- }
- }
- }()
-
- timeDiff := time.Since(t.Time())
- lastRequestTimeFromAnn := lastRequestTimeFromAnnotation(t.leaseClient, t.log.WithName("lastRequestTimeFromAnnotation"))
- if lastRequestTimeFromAnn == nil {
- if err := status.UpdateLastRequestTimestmap(t.Time()); err != nil {
- logger.Error(err, "failed to annotate deployment for lastRequestTime")
- } else {
- logger.Info("initialized lastRequestTimestamp", "time", t.Time())
- }
- continue
- }
-
- switch {
- case timeDiff > idleDeadline:
- err := fmt.Errorf("webhook hasn't received requests in %v, updating Kyverno to verify webhook status", idleDeadline.String())
- logger.Error(err, "webhook check failed", "time", t.Time(), "lastRequestTimestamp", lastRequestTimeFromAnn)
-
- // update deployment to renew lastSeenRequestTime
- if err := status.failure(); err != nil {
- logger.Error(err, "failed to annotate deployment webhook status to failure")
-
- if err := register.Register(); err != nil {
- logger.Error(err, "Failed to register webhooks")
- }
- }
-
- continue
-
- case timeDiff > 2*idleCheckInterval:
- if skipWebhookCheck(register, logger.WithName("skipWebhookCheck")) {
- logger.Info("skip validating webhook status, Kyverno is in rolling update")
- continue
- }
-
- if t.Time().Before(*lastRequestTimeFromAnn) {
- t.SetTime(*lastRequestTimeFromAnn)
- logger.V(3).Info("updated in-memory timestamp", "time", lastRequestTimeFromAnn)
- }
- }
-
- idleT := time.Since(*lastRequestTimeFromAnn)
- if idleT > idleCheckInterval {
- if t.Time().After(*lastRequestTimeFromAnn) {
- logger.V(3).Info("updating annotation lastRequestTimestamp with the latest in-memory timestamp", "time", t.Time(), "lastRequestTimestamp", lastRequestTimeFromAnn)
- if err := status.UpdateLastRequestTimestmap(t.Time()); err != nil {
- logger.Error(err, "failed to update lastRequestTimestamp annotation")
- }
- }
- }
-
- // if the status was false before then we update it to true
- // send request to update the Kyverno deployment
- if err := status.success(); err != nil {
- logger.Error(err, "failed to annotate deployment webhook status to success")
- }
-
- case <-ctx.Done():
- // handler termination signal
- logger.V(2).Info("stopping webhook monitor")
- return
- }
- }
-}
-
-func registerWebhookIfNotPresent(register *Register, logger logr.Logger) error {
- if skipWebhookCheck(register, logger.WithName("skipWebhookCheck")) {
- logger.Info("skip validating webhook status, Kyverno is in rolling update")
- return nil
- }
-
- if err := register.Check(); err != nil {
- logger.Error(err, "missing webhooks")
-
- if err := register.Register(); err != nil {
- return errors.Wrap(err, "failed to register webhooks")
- }
- }
-
- return nil
-}
-
-func lastRequestTimeFromAnnotation(leaseClient coordinationv1.LeaseInterface, logger logr.Logger) *time.Time {
- lease, err := leaseClient.Get(context.TODO(), "kyverno", metav1.GetOptions{})
- if err != nil {
- logger.Info("Lease 'kyverno' not found. Starting clean-up...")
- }
-
- timeStamp := lease.GetAnnotations()
- if timeStamp == nil {
- logger.Info("timestamp not set in the annotation, setting")
- return nil
- }
-
- annTime, err := time.Parse(time.RFC3339, timeStamp[annLastRequestTime])
- if err != nil {
- logger.Error(err, "failed to parse timestamp annotation", "timeStamp", timeStamp[annLastRequestTime])
- return nil
- }
-
- return &annTime
-}
-
-// skipWebhookCheck returns true if Kyverno is in rolling update
-func skipWebhookCheck(register *Register, logger logr.Logger) bool {
- deploy, err := register.GetKubePolicyDeployment()
- if err != nil {
- logger.Info("unable to get Kyverno deployment", "reason", err.Error())
- return false
- }
- return tls.IsKyvernoInRollingUpdate(deploy)
-}
diff --git a/pkg/webhookconfig/registration.go b/pkg/webhookconfig/registration.go
deleted file mode 100644
index 71564ce6f5..0000000000
--- a/pkg/webhookconfig/registration.go
+++ /dev/null
@@ -1,653 +0,0 @@
-package webhookconfig
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "reflect"
- "strings"
- "sync"
- "time"
-
- "github.com/go-logr/logr"
- kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
- "github.com/kyverno/kyverno/pkg/client/clientset/versioned"
- kyvernov1informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1"
- "github.com/kyverno/kyverno/pkg/clients/dclient"
- "github.com/kyverno/kyverno/pkg/config"
- "github.com/kyverno/kyverno/pkg/metrics"
- tlsutils "github.com/kyverno/kyverno/pkg/tls"
- "github.com/kyverno/kyverno/pkg/utils"
- "github.com/pkg/errors"
- admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
- errorsapi "k8s.io/apimachinery/pkg/api/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- admissionregistrationv1informers "k8s.io/client-go/informers/admissionregistration/v1"
- appsv1informers "k8s.io/client-go/informers/apps/v1"
- "k8s.io/client-go/kubernetes"
- admissionregistrationv1listers "k8s.io/client-go/listers/admissionregistration/v1"
- appsv1listers "k8s.io/client-go/listers/apps/v1"
- rest "k8s.io/client-go/rest"
-)
-
-const (
- kindMutating string = "MutatingWebhookConfiguration"
- kindValidating string = "ValidatingWebhookConfiguration"
-)
-
-// Register manages webhook registration. There are five webhooks:
-// 1. Policy Validation
-// 2. Policy Mutation
-// 3. Resource Validation
-// 4. Resource Mutation
-// 5. Webhook Status Mutation
-type Register struct {
- // clients
- kubeClient kubernetes.Interface
- kyvernoClient versioned.Interface
- clientConfig *rest.Config
-
- // listers
- mwcLister admissionregistrationv1listers.MutatingWebhookConfigurationLister
- vwcLister admissionregistrationv1listers.ValidatingWebhookConfigurationLister
- kDeplLister appsv1listers.DeploymentLister
-
- metricsConfig metrics.MetricsConfigManager
-
- // channels
- stopCh <-chan struct{}
- UpdateWebhookChan chan bool
- createDefaultWebhook chan string
-
- serverIP string // when running outside a cluster
- timeoutSeconds int32
- log logr.Logger
- autoUpdateWebhooks bool
-
- // manage implements methods to manage webhook configurations
- manage
-}
-
-// NewRegister creates new Register instance
-func NewRegister(
- ctx context.Context,
- clientConfig *rest.Config,
- client dclient.Interface,
- kubeClient kubernetes.Interface,
- kyvernoClient versioned.Interface,
- mwcInformer admissionregistrationv1informers.MutatingWebhookConfigurationInformer,
- vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer,
- kDeplInformer appsv1informers.DeploymentInformer,
- pInformer kyvernov1informers.ClusterPolicyInformer,
- npInformer kyvernov1informers.PolicyInformer,
- metricsConfig metrics.MetricsConfigManager,
- serverIP string,
- webhookTimeout int32,
- autoUpdateWebhooks bool,
- log logr.Logger,
-) *Register {
- register := &Register{
- clientConfig: clientConfig,
- kubeClient: kubeClient,
- kyvernoClient: kyvernoClient,
- mwcLister: mwcInformer.Lister(),
- vwcLister: vwcInformer.Lister(),
- kDeplLister: kDeplInformer.Lister(),
- metricsConfig: metricsConfig,
- UpdateWebhookChan: make(chan bool),
- createDefaultWebhook: make(chan string),
- stopCh: ctx.Done(),
- serverIP: serverIP,
- timeoutSeconds: webhookTimeout,
- log: log.WithName("Register"),
- autoUpdateWebhooks: autoUpdateWebhooks,
- }
-
- register.manage = newWebhookConfigManager(
- ctx,
- client.Discovery(),
- kubeClient,
- kyvernoClient,
- pInformer,
- npInformer,
- mwcInformer,
- vwcInformer,
- metricsConfig,
- serverIP,
- register.autoUpdateWebhooks,
- register.createDefaultWebhook,
- log.WithName("WebhookConfigManager"),
- )
-
- return register
-}
-
-// Register clean up the old webhooks and re-creates admission webhooks configs on cluster
-func (wrc *Register) Register() error {
- logger := wrc.log
- if wrc.serverIP != "" {
- logger.Info("Registering webhook", "url", fmt.Sprintf("https://%s", wrc.serverIP))
- } else {
- if err := wrc.checkEndpoint(); err != nil {
- return err
- }
- }
- caData := wrc.readCaData()
- if caData == nil {
- return errors.New("Unable to extract CA data from configuration")
- }
- var errors []string
- if err := wrc.createVerifyMutatingWebhookConfiguration(caData); err != nil {
- errors = append(errors, err.Error())
- }
- if err := wrc.createPolicyValidatingWebhookConfiguration(caData); err != nil {
- errors = append(errors, err.Error())
- }
- if err := wrc.createPolicyMutatingWebhookConfiguration(caData); err != nil {
- errors = append(errors, err.Error())
- }
- if err := wrc.createResourceValidatingWebhookConfiguration(caData); err != nil {
- errors = append(errors, err.Error())
- }
- if err := wrc.createResourceMutatingWebhookConfiguration(caData); err != nil {
- errors = append(errors, err.Error())
- }
- if len(errors) > 0 {
- return fmt.Errorf("%s", strings.Join(errors, ","))
- }
- go wrc.manage.start()
- return nil
-}
-
-// Check returns an error if any of the webhooks are not configured
-func (wrc *Register) Check() error {
- if _, err := wrc.mwcLister.Get(getVerifyMutatingWebhookConfigName(wrc.serverIP)); err != nil {
- return err
- }
- if _, err := wrc.mwcLister.Get(getResourceMutatingWebhookConfigName(wrc.serverIP)); err != nil {
- return err
- }
- if _, err := wrc.vwcLister.Get(getResourceValidatingWebhookConfigName(wrc.serverIP)); err != nil {
- return err
- }
- if _, err := wrc.mwcLister.Get(getPolicyMutatingWebhookConfigName(wrc.serverIP)); err != nil {
- return err
- }
- if _, err := wrc.vwcLister.Get(getPolicyValidatingWebhookConfigName(wrc.serverIP)); err != nil {
- return err
- }
- return nil
-}
-
-// Remove removes all webhook configurations
-func (wrc *Register) Remove(cleanupKyvernoResource bool, wg *sync.WaitGroup) {
- defer wg.Done()
- // delete Lease object to let init container do the cleanup
- if err := wrc.kubeClient.CoordinationV1().Leases(config.KyvernoNamespace()).Delete(context.TODO(), "kyvernopre-lock", metav1.DeleteOptions{}); err != nil && errorsapi.IsNotFound(err) {
- wrc.metricsConfig.RecordClientQueries(metrics.ClientDelete, metrics.KubeClient, "Lease", config.KyvernoNamespace())
- wrc.log.WithName("cleanup").Error(err, "failed to clean up Lease lock")
- }
- if cleanupKyvernoResource {
- wrc.removeWebhookConfigurations()
- }
-}
-
-func (wrc *Register) ResetPolicyStatus(kyvernoInTermination bool, wg *sync.WaitGroup) {
- defer wg.Done()
-
- if !kyvernoInTermination {
- return
- }
-
- logger := wrc.log.WithName("ResetPolicyStatus")
- cpols, err := wrc.kyvernoClient.KyvernoV1().ClusterPolicies().List(context.TODO(), metav1.ListOptions{})
- if err == nil {
- for _, item := range cpols.Items {
- cpol := item
- cpol.Status.SetReady(false)
- if _, err := wrc.kyvernoClient.KyvernoV1().ClusterPolicies().UpdateStatus(context.TODO(), &cpol, metav1.UpdateOptions{}); err != nil {
- logger.Error(err, "failed to set ClusterPolicy status READY=false", "name", cpol.GetName())
- }
- }
- } else {
- logger.Error(err, "failed to list clusterpolicies")
- }
-
- pols, err := wrc.kyvernoClient.KyvernoV1().Policies(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
- if err == nil {
- for _, item := range pols.Items {
- pol := item
- pol.Status.SetReady(false)
- if _, err := wrc.kyvernoClient.KyvernoV1().Policies(pol.GetNamespace()).UpdateStatus(context.TODO(), &pol, metav1.UpdateOptions{}); err != nil {
- logger.Error(err, "failed to set Policy status READY=false", "namespace", pol.GetNamespace(), "name", pol.GetName())
- }
- }
- } else {
- logger.Error(err, "failed to list namespaced policies")
- }
-}
-
-// GetWebhookTimeOut returns the value of webhook timeout
-func (wrc *Register) GetWebhookTimeOut() time.Duration {
- return time.Duration(wrc.timeoutSeconds)
-}
-
-// UpdateWebhookConfigurations updates resource webhook configurations dynamically
-// based on the UPDATEs of Kyverno ConfigMap defined in INIT_CONFIG env
-//
-// it currently updates namespaceSelector only, can be extend to update other fields
-// +deprecated
-func (wrc *Register) UpdateWebhookConfigurations(configHandler config.Configuration) {
- logger := wrc.log.WithName("UpdateWebhookConfigurations")
- for {
- <-wrc.UpdateWebhookChan
- logger.V(4).Info("received the signal to update webhook configurations")
-
- retry := false
- if wrc.serverIP != "" {
- deploy, err := wrc.GetKubePolicyDeployment()
- if err != nil {
- retry = true
- } else {
- if tlsutils.IsKyvernoInRollingUpdate(deploy) {
- retry = true
- }
- }
- }
-
- if !retry {
- webhookCfgs := configHandler.GetWebhooks()
- webhookCfg := config.WebhookConfig{}
- if len(webhookCfgs) > 0 {
- webhookCfg = webhookCfgs[0]
- }
-
- if err := wrc.updateResourceMutatingWebhookConfiguration(webhookCfg); err != nil {
- logger.Error(err, "unable to update mutatingWebhookConfigurations", "name", getResourceMutatingWebhookConfigName(wrc.serverIP))
- retry = true
- }
-
- if err := wrc.updateResourceValidatingWebhookConfiguration(webhookCfg); err != nil {
- logger.Error(err, "unable to update validatingWebhookConfigurations", "name", getResourceValidatingWebhookConfigName(wrc.serverIP))
- retry = true
- }
- }
-
- if retry {
- go func() {
- time.Sleep(1 * time.Second)
- select {
- case wrc.UpdateWebhookChan <- true:
- return
- default:
- return
- }
- }()
- }
- }
-}
-
-func (wrc *Register) ValidateWebhookConfigurations(namespace, name string) error {
- logger := wrc.log.WithName("ValidateWebhookConfigurations")
- cm, err := wrc.kubeClient.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metav1.GetOptions{})
- wrc.metricsConfig.RecordClientQueries(metrics.ClientGet, metrics.KubeClient, "ConfigMap", namespace)
- if err != nil {
- logger.Error(err, "unable to fetch ConfigMap", "namespace", namespace, "name", name)
- return nil
- }
- webhooks, ok := cm.Data["webhooks"]
- if !ok {
- logger.V(4).Info("webhook configurations not defined")
- return nil
- }
- webhookCfgs := make([]config.WebhookConfig, 0, 10)
- return json.Unmarshal([]byte(webhooks), &webhookCfgs)
-}
-
-func (wrc *Register) createMutatingWebhookConfiguration(config *admissionregistrationv1.MutatingWebhookConfiguration) error {
- logger := wrc.log.WithValues("kind", kindMutating, "name", config.Name)
-
- wrc.metricsConfig.RecordClientQueries(metrics.ClientCreate, metrics.KubeClient, kindMutating, "")
- if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{}); err != nil {
- if errorsapi.IsAlreadyExists(err) {
- logger.V(6).Info("resource mutating webhook configuration already exists", "name", config.Name)
- return wrc.updateMutatingWebhookConfiguration(config)
- }
- logger.Error(err, "failed to create resource mutating webhook configuration", "name", config.Name)
- return err
- }
- logger.Info("created webhook")
- return nil
-}
-
-func (wrc *Register) createValidatingWebhookConfiguration(config *admissionregistrationv1.ValidatingWebhookConfiguration) error {
- logger := wrc.log.WithValues("kind", kindValidating, "name", config.Name)
-
- wrc.metricsConfig.RecordClientQueries(metrics.ClientCreate, metrics.KubeClient, kindValidating, "")
- if _, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{}); err != nil {
- if errorsapi.IsAlreadyExists(err) {
- logger.V(6).Info("resource validating webhook configuration already exists", "name", config.Name)
- return wrc.updateValidatingWebhookConfiguration(config)
- }
- logger.Error(err, "failed to create resource validating webhook configuration", "name", config.Name)
- return err
- }
- logger.Info("created webhook")
- return nil
-}
-
-func (wrc *Register) createResourceMutatingWebhookConfiguration(caData []byte) error {
- owner := wrc.constructOwner()
- var config *admissionregistrationv1.MutatingWebhookConfiguration
- if wrc.serverIP != "" {
- config = constructDefaultDebugMutatingWebhookConfig(wrc.serverIP, caData, wrc.timeoutSeconds, wrc.autoUpdateWebhooks, owner)
- } else {
- config = constructDefaultMutatingWebhookConfig(caData, wrc.timeoutSeconds, wrc.autoUpdateWebhooks, owner)
- }
- return wrc.createMutatingWebhookConfiguration(config)
-}
-
-func (wrc *Register) createResourceValidatingWebhookConfiguration(caData []byte) error {
- owner := wrc.constructOwner()
- var config *admissionregistrationv1.ValidatingWebhookConfiguration
- if wrc.serverIP != "" {
- config = constructDefaultDebugValidatingWebhookConfig(wrc.serverIP, caData, wrc.timeoutSeconds, wrc.autoUpdateWebhooks, owner)
- } else {
- config = constructDefaultValidatingWebhookConfig(caData, wrc.timeoutSeconds, wrc.autoUpdateWebhooks, owner)
- }
- return wrc.createValidatingWebhookConfiguration(config)
-}
-
-func (wrc *Register) createPolicyValidatingWebhookConfiguration(caData []byte) error {
- owner := wrc.constructOwner()
- var config *admissionregistrationv1.ValidatingWebhookConfiguration
- if wrc.serverIP != "" {
- config = constructDebugPolicyValidatingWebhookConfig(wrc.serverIP, caData, wrc.timeoutSeconds, owner)
- } else {
- config = constructPolicyValidatingWebhookConfig(caData, wrc.timeoutSeconds, owner)
- }
- return wrc.createValidatingWebhookConfiguration(config)
-}
-
-func (wrc *Register) createPolicyMutatingWebhookConfiguration(caData []byte) error {
- owner := wrc.constructOwner()
- var config *admissionregistrationv1.MutatingWebhookConfiguration
- if wrc.serverIP != "" {
- config = constructDebugPolicyMutatingWebhookConfig(wrc.serverIP, caData, wrc.timeoutSeconds, owner)
- } else {
- config = constructPolicyMutatingWebhookConfig(caData, wrc.timeoutSeconds, owner)
- }
- return wrc.createMutatingWebhookConfiguration(config)
-}
-
-func (wrc *Register) createVerifyMutatingWebhookConfiguration(caData []byte) error {
- owner := wrc.constructOwner()
- var config *admissionregistrationv1.MutatingWebhookConfiguration
- if wrc.serverIP != "" {
- config = constructDebugVerifyMutatingWebhookConfig(wrc.serverIP, caData, wrc.timeoutSeconds, owner)
- } else {
- config = constructVerifyMutatingWebhookConfig(caData, wrc.timeoutSeconds, owner)
- }
- return wrc.createMutatingWebhookConfiguration(config)
-}
-
-func (wrc *Register) checkEndpoint() error {
- endpoint, err := wrc.kubeClient.CoreV1().Endpoints(config.KyvernoNamespace()).Get(context.TODO(), config.KyvernoServiceName(), metav1.GetOptions{})
- wrc.metricsConfig.RecordClientQueries(metrics.ClientGet, metrics.KubeClient, "EndPoint", config.KyvernoNamespace())
- if err != nil {
- return fmt.Errorf("failed to get endpoint %s/%s: %v", config.KyvernoNamespace(), config.KyvernoServiceName(), err)
- }
- deploy, err := wrc.GetKubePolicyDeployment()
- if err != nil {
- return err
- }
- if tlsutils.IsKyvernoInRollingUpdate(deploy) {
- return errors.New("kyverno is in rolling update, please update the timeout by setting the webhookRegistrationTimeout flag")
- }
- selector := &metav1.LabelSelector{
- MatchLabels: map[string]string{
- "app.kubernetes.io/name": kyvernov1.ValueKyvernoApp,
- },
- }
- pods, err := wrc.kubeClient.CoreV1().Pods(config.KyvernoNamespace()).List(context.TODO(), metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)})
- wrc.metricsConfig.RecordClientQueries(metrics.ClientList, metrics.KubeClient, "Pod", config.KyvernoNamespace())
- if err != nil {
- return fmt.Errorf("failed to list Kyverno Pod: %v", err)
- }
- ips := getHealthyPodsIP(pods.Items)
- if len(ips) == 0 {
- return fmt.Errorf("pod is not assigned to any node yet")
- }
- for _, subset := range endpoint.Subsets {
- if len(subset.Addresses) == 0 {
- continue
- }
- for _, addr := range subset.Addresses {
- if utils.ContainsString(ips, addr.IP) {
- wrc.log.V(2).Info("Endpoint ready", "ns", config.KyvernoNamespace(), "name", config.KyvernoServiceName())
- return nil
- }
- }
- }
- err = fmt.Errorf("endpoint not ready")
- wrc.log.V(3).Info(err.Error(), "ns", config.KyvernoNamespace(), "name", config.KyvernoServiceName())
- return err
-}
-
-func (wrc *Register) updateResourceValidatingWebhookConfiguration(webhookCfg config.WebhookConfig) error {
- resource, err := wrc.vwcLister.Get(getResourceValidatingWebhookConfigName(wrc.serverIP))
- if err != nil {
- return errors.Wrapf(err, "unable to get validatingWebhookConfigurations")
- }
- copy := resource.DeepCopy()
- for i := range copy.Webhooks {
- copy.Webhooks[i].ObjectSelector = webhookCfg.ObjectSelector
- copy.Webhooks[i].NamespaceSelector = webhookCfg.NamespaceSelector
- }
- if reflect.DeepEqual(resource.Webhooks, copy.Webhooks) {
- wrc.log.V(4).Info("namespaceSelector unchanged, skip updating validatingWebhookConfigurations")
- return nil
- }
- wrc.metricsConfig.RecordClientQueries(metrics.ClientUpdate, metrics.KubeClient, kindValidating, "")
- if _, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(context.TODO(), copy, metav1.UpdateOptions{}); err != nil {
- return err
- }
- wrc.log.V(3).Info("successfully updated validatingWebhookConfigurations", "name", getResourceMutatingWebhookConfigName(wrc.serverIP))
- return nil
-}
-
-func (wrc *Register) updateResourceMutatingWebhookConfiguration(webhookCfg config.WebhookConfig) error {
- resource, err := wrc.mwcLister.Get(getResourceMutatingWebhookConfigName(wrc.serverIP))
- if err != nil {
- return errors.Wrapf(err, "unable to get mutatingWebhookConfigurations")
- }
- copy := resource.DeepCopy()
- for i := range copy.Webhooks {
- copy.Webhooks[i].ObjectSelector = webhookCfg.ObjectSelector
- copy.Webhooks[i].NamespaceSelector = webhookCfg.NamespaceSelector
- }
- if reflect.DeepEqual(resource.Webhooks, copy.Webhooks) {
- wrc.log.V(4).Info("namespaceSelector unchanged, skip updating mutatingWebhookConfigurations")
- return nil
- }
-
- wrc.metricsConfig.RecordClientQueries(metrics.ClientUpdate, metrics.KubeClient, kindMutating, "")
- if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.TODO(), copy, metav1.UpdateOptions{}); err != nil {
- return err
- }
- wrc.log.V(3).Info("successfully updated mutatingWebhookConfigurations", "name", getResourceMutatingWebhookConfigName(wrc.serverIP))
- return nil
-}
-
-// updateMutatingWebhookConfiguration updates an existing MutatingWebhookConfiguration with the rules provided by
-// the targetConfig. If the targetConfig doesn't provide any rules, the existing rules will be preserved.
-func (wrc *Register) updateMutatingWebhookConfiguration(targetConfig *admissionregistrationv1.MutatingWebhookConfiguration) error {
- // Fetch the existing webhook.
- currentConfiguration, err := wrc.mwcLister.Get(targetConfig.Name)
- if err != nil {
- return fmt.Errorf("failed to get %s %s: %v", kindMutating, targetConfig.Name, err)
- }
- // Create a map of the target webhooks.
- targetWebhooksMap := make(map[string]admissionregistrationv1.MutatingWebhook)
- for _, w := range targetConfig.Webhooks {
- targetWebhooksMap[w.Name] = w
- }
- // Update the webhooks.
- newWebhooks := make([]admissionregistrationv1.MutatingWebhook, 0)
- for _, w := range currentConfiguration.Webhooks {
- target, exist := targetWebhooksMap[w.Name]
- if !exist {
- continue
- }
- delete(targetWebhooksMap, w.Name)
- // Update the webhook configuration
- w.ClientConfig.URL = target.ClientConfig.URL
- w.ClientConfig.Service = target.ClientConfig.Service
- w.ClientConfig.CABundle = target.ClientConfig.CABundle
- if target.Rules != nil {
- // If the target webhook has rule definitions override the current.
- w.Rules = target.Rules
- }
- newWebhooks = append(newWebhooks, w)
- }
- // Check if there are additional webhooks defined and add them.
- for _, w := range targetWebhooksMap {
- newWebhooks = append(newWebhooks, w)
- }
- // Update the current configuration.
- currentConfiguration.Webhooks = newWebhooks
- wrc.metricsConfig.RecordClientQueries(metrics.ClientUpdate, metrics.KubeClient, kindMutating, "")
- if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.TODO(), currentConfiguration, metav1.UpdateOptions{}); err != nil {
- return err
- }
- wrc.log.V(3).Info("successfully updated mutatingWebhookConfigurations", "name", targetConfig.Name)
- return nil
-}
-
-// updateValidatingWebhookConfiguration updates an existing ValidatingWebhookConfiguration with the rules provided by
-// the targetConfig. If the targetConfig doesn't provide any rules, the existing rules will be preserved.
-func (wrc *Register) updateValidatingWebhookConfiguration(targetConfig *admissionregistrationv1.ValidatingWebhookConfiguration) error {
- // Fetch the existing webhook.
- currentConfiguration, err := wrc.vwcLister.Get(targetConfig.Name)
- if err != nil {
- return fmt.Errorf("failed to get %s %s: %v", kindValidating, targetConfig.Name, err)
- }
- // Create a map of the target webhooks.
- targetWebhooksMap := make(map[string]admissionregistrationv1.ValidatingWebhook)
- for _, w := range targetConfig.Webhooks {
- targetWebhooksMap[w.Name] = w
- }
- // Update the webhooks.
- newWebhooks := make([]admissionregistrationv1.ValidatingWebhook, 0)
- for _, w := range currentConfiguration.Webhooks {
- target, exist := targetWebhooksMap[w.Name]
- if !exist {
- continue
- }
- delete(targetWebhooksMap, w.Name)
- // Update the webhook configuration
- w.ClientConfig.URL = target.ClientConfig.URL
- w.ClientConfig.Service = target.ClientConfig.Service
- w.ClientConfig.CABundle = target.ClientConfig.CABundle
- if target.Rules != nil {
- // If the target webhook has rule definitions override the current.
- w.Rules = target.Rules
- }
- newWebhooks = append(newWebhooks, w)
- }
- // Check if there are additional webhooks defined and add them.
- for _, w := range targetWebhooksMap {
- newWebhooks = append(newWebhooks, w)
- }
- // Update the current configuration.
- currentConfiguration.Webhooks = newWebhooks
- wrc.metricsConfig.RecordClientQueries(metrics.ClientUpdate, metrics.KubeClient, kindValidating, "")
- if _, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(context.TODO(), currentConfiguration, metav1.UpdateOptions{}); err != nil {
- return err
- }
- wrc.log.V(3).Info("successfully updated validatingWebhookConfigurations", "name", targetConfig.Name)
- return nil
-}
-
-func (wrc *Register) ShouldCleanupKyvernoResource() bool {
- logger := wrc.log.WithName("cleanupKyvernoResource")
- deploy, err := wrc.kubeClient.AppsV1().Deployments(config.KyvernoNamespace()).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{})
- wrc.metricsConfig.RecordClientQueries(metrics.ClientGet, metrics.KubeClient, "Deployment", config.KyvernoNamespace())
- if err != nil {
- if errorsapi.IsNotFound(err) {
- logger.Info("Kyverno deployment not found, cleanup Kyverno resources")
- return true
- }
- logger.Error(err, "failed to get deployment, not cleaning up kyverno resources")
- return false
- }
- if deploy.GetDeletionTimestamp() != nil {
- logger.Info("Kyverno is terminating, cleanup Kyverno resources")
- return true
- }
- if deploy.Spec.Replicas != nil && *deploy.Spec.Replicas == 0 {
- logger.Info("Kyverno is scaled to zero, cleanup Kyverno resources")
- return true
- }
- logger.Info("updating Kyverno Pod, won't clean up Kyverno resources")
- return false
-}
-
-func (wrc *Register) removeWebhookConfigurations() {
- startTime := time.Now()
- wrc.log.V(3).Info("deleting all webhook configurations")
- defer wrc.log.V(4).Info("removed webhook configurations", "processingTime", time.Since(startTime).String())
- var wg sync.WaitGroup
- wg.Add(5)
- go wrc.removeResourceMutatingWebhookConfiguration(&wg)
- go wrc.removeResourceValidatingWebhookConfiguration(&wg)
- go wrc.removePolicyMutatingWebhookConfiguration(&wg)
- go wrc.removePolicyValidatingWebhookConfiguration(&wg)
- go wrc.removeVerifyWebhookMutatingWebhookConfig(&wg)
- wg.Wait()
-}
-
-func (wrc *Register) removeResourceMutatingWebhookConfiguration(wg *sync.WaitGroup) {
- defer wg.Done()
- wrc.removeMutatingWebhookConfiguration(getResourceMutatingWebhookConfigName(wrc.serverIP))
-}
-
-func (wrc *Register) removeResourceValidatingWebhookConfiguration(wg *sync.WaitGroup) {
- defer wg.Done()
- wrc.removeValidatingWebhookConfiguration(getResourceValidatingWebhookConfigName(wrc.serverIP))
-}
-
-func (wrc *Register) removePolicyMutatingWebhookConfiguration(wg *sync.WaitGroup) {
- defer wg.Done()
- wrc.removeMutatingWebhookConfiguration(getPolicyMutatingWebhookConfigName(wrc.serverIP))
-}
-
-func (wrc *Register) removePolicyValidatingWebhookConfiguration(wg *sync.WaitGroup) {
- defer wg.Done()
- wrc.removeValidatingWebhookConfiguration(getPolicyValidatingWebhookConfigName(wrc.serverIP))
-}
-
-func (wrc *Register) removeVerifyWebhookMutatingWebhookConfig(wg *sync.WaitGroup) {
- defer wg.Done()
- wrc.removeMutatingWebhookConfiguration(getVerifyMutatingWebhookConfigName(wrc.serverIP))
-}
-
-func (wrc *Register) removeMutatingWebhookConfiguration(name string) {
- logger := wrc.log.WithValues("kind", kindMutating, "name", name)
- if err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil && !errorsapi.IsNotFound(err) {
- logger.Error(err, "failed to delete the mutating webhook configuration")
- } else {
- logger.Info("webhook configuration deleted")
- }
- wrc.metricsConfig.RecordClientQueries(metrics.ClientDelete, metrics.KubeClient, kindMutating, "")
-}
-
-func (wrc *Register) removeValidatingWebhookConfiguration(name string) {
- logger := wrc.log.WithValues("kind", kindValidating, "name", name)
- if err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil && !errorsapi.IsNotFound(err) {
- logger.Error(err, "failed to delete the validating webhook configuration")
- } else {
- logger.Info("webhook configuration deleted")
- }
- wrc.metricsConfig.RecordClientQueries(metrics.ClientDelete, metrics.KubeClient, kindValidating, "")
-}
diff --git a/pkg/webhookconfig/status.go b/pkg/webhookconfig/status.go
deleted file mode 100644
index 4c4c49da5b..0000000000
--- a/pkg/webhookconfig/status.go
+++ /dev/null
@@ -1,134 +0,0 @@
-package webhookconfig
-
-import (
- "context"
- "fmt"
- "time"
-
- "github.com/go-logr/logr"
- kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
- "github.com/kyverno/kyverno/pkg/config"
- "github.com/kyverno/kyverno/pkg/event"
- "github.com/pkg/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1"
-)
-
-const (
- leaseName string = "kyverno"
- annWebhookStatus string = "kyverno.io/webhookActive"
- annLastRequestTime string = "kyverno.io/last-request-time"
-)
-
-// statusControl controls the webhook status
-type statusControl struct {
- eventGen event.Interface
- log logr.Logger
- leaseClient coordinationv1.LeaseInterface
-}
-
-// success ...
-func (vc statusControl) success() error {
- return vc.setStatus("true")
-}
-
-// failure ...
-func (vc statusControl) failure() error {
- return vc.setStatus("false")
-}
-
-// NewStatusControl creates a new webhook status control
-func newStatusControl(leaseClient coordinationv1.LeaseInterface, eventGen event.Interface, log logr.Logger) *statusControl {
- return &statusControl{
- eventGen: eventGen,
- log: log,
- leaseClient: leaseClient,
- }
-}
-
-func (vc statusControl) setStatus(status string) error {
- logger := vc.log.WithValues("name", leaseName, "namespace", config.KyvernoNamespace())
- var ann map[string]string
- var err error
-
- lease, err := vc.leaseClient.Get(context.TODO(), "kyverno", metav1.GetOptions{})
- if err != nil {
- vc.log.WithName("UpdateLastRequestTimestmap").Error(err, "Lease 'kyverno' not found. Starting clean-up...")
- return err
- }
-
- ann = lease.GetAnnotations()
- if ann == nil {
- ann = map[string]string{}
- ann[annWebhookStatus] = status
- }
-
- leaseStatus, ok := ann[annWebhookStatus]
- if ok {
- if leaseStatus == status {
- logger.V(4).Info(fmt.Sprintf("annotation %s already set to '%s'", annWebhookStatus, status))
- return nil
- }
- }
-
- ann[annWebhookStatus] = status
- lease.SetAnnotations(ann)
-
- _, err = vc.leaseClient.Update(context.TODO(), lease, metav1.UpdateOptions{})
- if err != nil {
- return errors.Wrapf(err, "key %s, val %s", annWebhookStatus, status)
- }
-
- logger.Info("updated lease annotation", "key", annWebhookStatus, "val", status)
-
- // create event on kyverno deployment
- createStatusUpdateEvent(status, vc.eventGen)
- return nil
-}
-
-func createStatusUpdateEvent(status string, eventGen event.Interface) {
- e := event.Info{}
- e.Kind = "Lease"
- e.Namespace = config.KyvernoNamespace()
- e.Name = leaseName
- e.Reason = "Update"
- e.Message = fmt.Sprintf("admission control webhook active status changed to %s", status)
- eventGen.Add(e)
-}
-
-func (vc statusControl) UpdateLastRequestTimestmap(new time.Time) error {
- lease, err := vc.leaseClient.Get(context.TODO(), leaseName, metav1.GetOptions{})
- if err != nil {
- vc.log.WithName("UpdateLastRequestTimestmap").Error(err, "Lease 'kyverno' not found. Starting clean-up...")
- return err
- }
-
- // add label to lease
- label := lease.GetLabels()
- if len(label) == 0 {
- label = make(map[string]string)
- label["app.kubernetes.io/name"] = kyvernov1.ValueKyvernoApp
- }
- lease.SetLabels(label)
-
- annotation := lease.GetAnnotations()
- if annotation == nil {
- annotation = make(map[string]string)
- }
-
- t, err := new.MarshalText()
- if err != nil {
- return errors.Wrap(err, "failed to marshal timestamp")
- }
-
- annotation[annLastRequestTime] = string(t)
- lease.SetAnnotations(annotation)
-
- // update annotations in lease
- _, err = vc.leaseClient.Update(context.TODO(), lease, metav1.UpdateOptions{})
- if err != nil {
- return errors.Wrapf(err, "failed to update annotation %s for deployment %s in namespace %s", annLastRequestTime, lease.GetName(), lease.GetNamespace())
- }
-
- return nil
-}
diff --git a/pkg/webhooks/handlers/admission.go b/pkg/webhooks/handlers/admission.go
index a1fb220cd1..950a7af82c 100644
--- a/pkg/webhooks/handlers/admission.go
+++ b/pkg/webhooks/handlers/admission.go
@@ -11,7 +11,6 @@ import (
"github.com/kyverno/kyverno/pkg/config"
"github.com/kyverno/kyverno/pkg/tracing"
admissionutils "github.com/kyverno/kyverno/pkg/utils/admission"
- "github.com/kyverno/kyverno/pkg/webhookconfig"
"go.opentelemetry.io/otel/attribute"
admissionv1 "k8s.io/api/admission/v1"
)
@@ -99,9 +98,8 @@ func Filter(c config.Configuration, inner AdmissionHandler) AdmissionHandler {
}
}
-func Verify(m *webhookconfig.Monitor) AdmissionHandler {
+func Verify() AdmissionHandler {
return func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
- logger.V(6).Info("incoming request", "last admission request timestamp", m.Time())
return admissionutils.Response(true)
}
}
diff --git a/pkg/webhooks/handlers/monitor.go b/pkg/webhooks/handlers/monitor.go
deleted file mode 100644
index d055648834..0000000000
--- a/pkg/webhooks/handlers/monitor.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package handlers
-
-import (
- "net/http"
- "time"
-
- "github.com/kyverno/kyverno/pkg/webhookconfig"
-)
-
-func Monitor(m *webhookconfig.Monitor, inner http.HandlerFunc) http.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) {
- m.SetTime(time.Now())
- inner(w, r)
- }
-}
diff --git a/pkg/webhooks/handlers/probe.go b/pkg/webhooks/handlers/probe.go
index c47ffc76b3..70629cd0fd 100644
--- a/pkg/webhooks/handlers/probe.go
+++ b/pkg/webhooks/handlers/probe.go
@@ -2,10 +2,10 @@ package handlers
import "net/http"
-func Probe(check func() error) http.HandlerFunc {
+func Probe(check func() bool) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if check != nil {
- if err := check(); err != nil {
+ if !check() {
w.WriteHeader(http.StatusInternalServerError)
}
}
diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go
index 09caff1e63..71ed206eef 100644
--- a/pkg/webhooks/server.go
+++ b/pkg/webhooks/server.go
@@ -5,7 +5,6 @@ import (
"crypto/tls"
"fmt"
"net/http"
- "sync"
"time"
"github.com/go-logr/logr"
@@ -15,9 +14,14 @@ import (
"github.com/kyverno/kyverno/pkg/toggle"
"github.com/kyverno/kyverno/pkg/utils"
admissionutils "github.com/kyverno/kyverno/pkg/utils/admission"
- "github.com/kyverno/kyverno/pkg/webhookconfig"
+ controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
+ runtimeutils "github.com/kyverno/kyverno/pkg/utils/runtime"
"github.com/kyverno/kyverno/pkg/webhooks/handlers"
admissionv1 "k8s.io/api/admission/v1"
+ admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+ coordinationv1 "k8s.io/api/coordination/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
@@ -45,9 +49,12 @@ type ResourceHandlers interface {
}
type server struct {
- server *http.Server
- webhookRegister *webhookconfig.Register
- cleanUp chan struct{}
+ server *http.Server
+ runtime runtimeutils.Runtime
+ mwcClient controllerutils.DeleteClient[*admissionregistrationv1.MutatingWebhookConfiguration]
+ vwcClient controllerutils.DeleteClient[*admissionregistrationv1.ValidatingWebhookConfiguration]
+ leaseClient controllerutils.DeleteClient[*coordinationv1.Lease]
+ cleanUp chan struct{}
}
type TlsProvider func() ([]byte, []byte, error)
@@ -56,22 +63,24 @@ type TlsProvider func() ([]byte, []byte, error)
func NewServer(
policyHandlers PolicyHandlers,
resourceHandlers ResourceHandlers,
- tlsProvider TlsProvider,
configuration config.Configuration,
- register *webhookconfig.Register,
- monitor *webhookconfig.Monitor,
+ tlsProvider TlsProvider,
+ mwcClient controllerutils.DeleteClient[*admissionregistrationv1.MutatingWebhookConfiguration],
+ vwcClient controllerutils.DeleteClient[*admissionregistrationv1.ValidatingWebhookConfiguration],
+ leaseClient controllerutils.DeleteClient[*coordinationv1.Lease],
+ runtime runtimeutils.Runtime,
) Server {
mux := httprouter.New()
resourceLogger := logger.WithName("resource")
policyLogger := logger.WithName("policy")
verifyLogger := logger.WithName("verify")
- registerWebhookHandlers(resourceLogger.WithName("mutate"), mux, config.MutatingWebhookServicePath, monitor, configuration, resourceHandlers.Mutate)
- registerWebhookHandlers(resourceLogger.WithName("validate"), mux, config.ValidatingWebhookServicePath, monitor, configuration, resourceHandlers.Validate)
- mux.HandlerFunc("POST", config.PolicyMutatingWebhookServicePath, admission(policyLogger.WithName("mutate"), monitor, filter(configuration, policyHandlers.Mutate)))
- mux.HandlerFunc("POST", config.PolicyValidatingWebhookServicePath, admission(policyLogger.WithName("validate"), monitor, filter(configuration, policyHandlers.Validate)))
- mux.HandlerFunc("POST", config.VerifyMutatingWebhookServicePath, admission(verifyLogger.WithName("mutate"), monitor, handlers.Verify(monitor)))
- mux.HandlerFunc("GET", config.LivenessServicePath, handlers.Probe(register.Check))
- mux.HandlerFunc("GET", config.ReadinessServicePath, handlers.Probe(nil))
+ registerWebhookHandlers(resourceLogger.WithName("mutate"), mux, config.MutatingWebhookServicePath, configuration, resourceHandlers.Mutate)
+ registerWebhookHandlers(resourceLogger.WithName("validate"), mux, config.ValidatingWebhookServicePath, configuration, resourceHandlers.Validate)
+ mux.HandlerFunc("POST", config.PolicyMutatingWebhookServicePath, admission(policyLogger.WithName("mutate"), filter(configuration, policyHandlers.Mutate)))
+ mux.HandlerFunc("POST", config.PolicyValidatingWebhookServicePath, admission(policyLogger.WithName("validate"), filter(configuration, policyHandlers.Validate)))
+ mux.HandlerFunc("POST", config.VerifyMutatingWebhookServicePath, admission(verifyLogger.WithName("mutate"), handlers.Verify()))
+ mux.HandlerFunc("GET", config.LivenessServicePath, handlers.Probe(runtime.IsLive))
+ mux.HandlerFunc("GET", config.ReadinessServicePath, handlers.Probe(runtime.IsReady))
return &server{
server: &http.Server{
Addr: ":9443",
@@ -94,8 +103,11 @@ func NewServer(
WriteTimeout: 30 * time.Second,
ReadHeaderTimeout: 30 * time.Second,
},
- webhookRegister: register,
- cleanUp: make(chan struct{}),
+ mwcClient: mwcClient,
+ vwcClient: vwcClient,
+ leaseClient: leaseClient,
+ runtime: runtime,
+ cleanUp: make(chan struct{}),
}
}
@@ -126,13 +138,29 @@ func (s *server) Cleanup() <-chan struct{} {
}
func (s *server) cleanup(ctx context.Context) {
- cleanupKyvernoResource := s.webhookRegister.ShouldCleanupKyvernoResource()
-
- var wg sync.WaitGroup
- wg.Add(2)
- go s.webhookRegister.Remove(cleanupKyvernoResource, &wg)
- go s.webhookRegister.ResetPolicyStatus(cleanupKyvernoResource, &wg)
- wg.Wait()
+ if s.runtime.IsGoingDown() {
+ deleteLease := func(name string) {
+ if err := s.leaseClient.Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
+ logger.Error(err, "failed to clean up lease", "name", name)
+ }
+ }
+ deleteVwc := func(name string) {
+ if err := s.vwcClient.Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
+ logger.Error(err, "failed to clean up validating webhook configuration", "name", name)
+ }
+ }
+ deleteMwc := func(name string) {
+ if err := s.mwcClient.Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
+ logger.Error(err, "failed to clean up mutating webhook configuration", "name", name)
+ }
+ }
+ deleteLease("kyvernopre-lock")
+ deleteVwc(config.ValidatingWebhookConfigurationName)
+ deleteVwc(config.PolicyValidatingWebhookConfigurationName)
+ deleteMwc(config.MutatingWebhookConfigurationName)
+ deleteMwc(config.PolicyMutatingWebhookConfigurationName)
+ deleteMwc(config.VerifyMutatingWebhookConfigurationName)
+ }
close(s.cleanUp)
}
@@ -162,31 +190,30 @@ func filter(configuration config.Configuration, inner handlers.AdmissionHandler)
return handlers.Filter(configuration, inner)
}
-func admission(logger logr.Logger, monitor *webhookconfig.Monitor, inner handlers.AdmissionHandler) http.HandlerFunc {
- return handlers.Monitor(monitor, handlers.Admission(logger, protect(inner)))
+func admission(logger logr.Logger, inner handlers.AdmissionHandler) http.HandlerFunc {
+ return handlers.Admission(logger, protect(inner))
}
func registerWebhookHandlers(
logger logr.Logger,
mux *httprouter.Router,
basePath string,
- monitor *webhookconfig.Monitor,
configuration config.Configuration,
handlerFunc func(logr.Logger, *admissionv1.AdmissionRequest, string, time.Time) *admissionv1.AdmissionResponse,
) {
- mux.HandlerFunc("POST", basePath, admission(logger, monitor, filter(
+ mux.HandlerFunc("POST", basePath, admission(logger, filter(
configuration,
func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
return handlerFunc(logger, request, "all", startTime)
})),
)
- mux.HandlerFunc("POST", basePath+"/fail", admission(logger, monitor, filter(
+ mux.HandlerFunc("POST", basePath+"/fail", admission(logger, filter(
configuration,
func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
return handlerFunc(logger, request, "fail", startTime)
})),
)
- mux.HandlerFunc("POST", basePath+"/ignore", admission(logger, monitor, filter(
+ mux.HandlerFunc("POST", basePath+"/ignore", admission(logger, filter(
configuration,
func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
return handlerFunc(logger, request, "ignore", startTime)