diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index 677c852d33..a18149de88 100755 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -238,6 +238,8 @@ func main() { clientConfig, client, pclient, + kubeInformer.Admissionregistration().V1().MutatingWebhookConfigurations(), + kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(), rCache, kubeKyvernoInformer.Apps().V1().Deployments(), kubeInformer.Core().V1().Namespaces(), diff --git a/pkg/resourcecache/main.go b/pkg/resourcecache/main.go index d4879e8298..8375cd0982 100644 --- a/pkg/resourcecache/main.go +++ b/pkg/resourcecache/main.go @@ -33,7 +33,7 @@ type resourceCache struct { log logr.Logger } -var KyvernoDefaultInformer = []string{"ConfigMap", "Deployment", "MutatingWebhookConfiguration", "ValidatingWebhookConfiguration"} +var KyvernoDefaultInformer = []string{} // NewResourceCache - initializes the ResourceCache func NewResourceCache(dclient *dclient.Client, dInformer dynamicinformer.DynamicSharedInformerFactory, logger logr.Logger) (ResourceCache, error) { diff --git a/pkg/webhookconfig/configmanager.go b/pkg/webhookconfig/configmanager.go index 79f1b41702..cd669fae40 100644 --- a/pkg/webhookconfig/configmanager.go +++ b/pkg/webhookconfig/configmanager.go @@ -2,6 +2,7 @@ package webhookconfig import ( "context" + "encoding/json" "fmt" "reflect" "strings" @@ -26,7 +27,9 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + adminformers "k8s.io/client-go/informers/admissionregistration/v1" informers "k8s.io/client-go/informers/core/v1" + admlisters "k8s.io/client-go/listers/admissionregistration/v1" listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" @@ -59,6 +62,8 @@ type webhookConfigManager struct { mutateInformer cache.SharedIndexInformer validateInformer cache.SharedIndexInformer + mutateLister admlisters.MutatingWebhookConfigurationLister + validateLister admlisters.ValidatingWebhookConfigurationLister mutateInformerSynced cache.InformerSynced validateInformerSynced cache.InformerSynced @@ -91,6 +96,8 @@ func newWebhookConfigManager( kyvernoClient *kyvernoclient.Clientset, pInformer kyvernoinformer.ClusterPolicyInformer, npInformer kyvernoinformer.PolicyInformer, + mwcInformer adminformers.MutatingWebhookConfigurationInformer, + vwcInformer adminformers.ValidatingWebhookConfigurationInformer, resCache resourcecache.ResourceCache, nsInformer informers.NamespaceInformer, serverIP string, @@ -123,13 +130,13 @@ func newWebhookConfigManager( m.pListerSynced = pInformer.Informer().HasSynced m.npListerSynced = npInformer.Informer().HasSynced - mutateCache, _ := m.resCache.GetGVRCache(kindMutating) - m.mutateInformer = mutateCache.GetInformer() - m.mutateInformerSynced = mutateCache.GetInformer().HasSynced + m.mutateInformer = mwcInformer.Informer() + m.mutateLister = mwcInformer.Lister() + m.mutateInformerSynced = mwcInformer.Informer().HasSynced - validateCache, _ := m.resCache.GetGVRCache(kindValidating) - m.validateInformer = validateCache.GetInformer() - m.validateInformerSynced = validateCache.GetInformer().HasSynced + m.validateInformer = vwcInformer.Informer() + m.validateLister = vwcInformer.Lister() + m.validateInformerSynced = vwcInformer.Informer().HasSynced return m } @@ -506,6 +513,7 @@ func (m *webhookConfigManager) buildWebhooks(namespace string) (res []*webhook, func (m *webhookConfigManager) updateWebhookConfig(webhooks []*webhook) error { logger := m.log.WithName("updateWebhookConfig") + webhooksMap := make(map[string]interface{}, len(webhooks)) for _, w := range webhooks { key := webhookKey(w.kind, string(w.failurePolicy)) @@ -532,16 +540,45 @@ func (m *webhookConfigManager) updateWebhookConfig(webhooks []*webhook) error { func (m *webhookConfigManager) getWebhook(webhookKind, webhookName string) (resourceWebhook *unstructured.Unstructured, err error) { get := func() error { - webhookCache, _ := m.resCache.GetGVRCache(webhookKind) + resourceWebhook = &unstructured.Unstructured{} + err = nil - resourceWebhook, err = webhookCache.Lister().Get(webhookName) - if err != nil && !apierrors.IsNotFound(err) { - return errors.Wrapf(err, "unable to get %s/%s", webhookKind, webhookName) - } else if apierrors.IsNotFound(err) { - m.createDefaultWebhook <- webhookKind - return err + var rawResc []byte + + switch webhookKind { + case kindMutating: + resourceWebhookTyped, err := m.mutateLister.Get(webhookName) + if err != nil && !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "unable to get %s/%s", webhookKind, webhookName) + } else if apierrors.IsNotFound(err) { + m.createDefaultWebhook <- webhookKind + return err + } + resourceWebhookTyped.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "admissionregistration.k8s.io/v1", Kind: kindMutating}) + rawResc, err = json.Marshal(resourceWebhookTyped) + if err != nil { + return err + } + case kindValidating: + resourceWebhookTyped, err := m.validateLister.Get(webhookName) + if err != nil && !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "unable to get %s/%s", webhookKind, webhookName) + } else if apierrors.IsNotFound(err) { + m.createDefaultWebhook <- webhookKind + return err + } + resourceWebhookTyped.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "admissionregistration.k8s.io/v1", Kind: kindValidating}) + rawResc, err = json.Marshal(resourceWebhookTyped) + if err != nil { + return err + } + default: + return fmt.Errorf("unknown webhook kind: must be '%v' or '%v'", kindMutating, kindValidating) } - return nil + + err = json.Unmarshal(rawResc, &resourceWebhook.Object) + + return err } retryGetWebhook := common.RetryFunc(time.Second, 10*time.Second, get, m.log) diff --git a/pkg/webhookconfig/registration.go b/pkg/webhookconfig/registration.go index 783c66176c..27ffc5e1e2 100644 --- a/pkg/webhookconfig/registration.go +++ b/pkg/webhookconfig/registration.go @@ -22,8 +22,11 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + adminformers "k8s.io/client-go/informers/admissionregistration/v1" informers "k8s.io/client-go/informers/apps/v1" coreinformers "k8s.io/client-go/informers/core/v1" + admlisters "k8s.io/client-go/listers/admissionregistration/v1" listers "k8s.io/client-go/listers/apps/v1" rest "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" @@ -41,9 +44,16 @@ const ( // 4. Resource Mutation // 5. Webhook Status Mutation type Register struct { - client *client.Client - clientConfig *rest.Config - resCache resourcecache.ResourceCache + client *client.Client + clientConfig *rest.Config + resCache resourcecache.ResourceCache + + mwcLister admlisters.MutatingWebhookConfigurationLister + vwcLister admlisters.ValidatingWebhookConfigurationLister + + mwcListerSynced func() bool + vwcListerSynced func() bool + serverIP string // when running outside a cluster timeoutSeconds int32 log logr.Logger @@ -66,6 +76,8 @@ func NewRegister( clientConfig *rest.Config, client *client.Client, kyvernoClient *kyvernoclient.Clientset, + mwcInformer adminformers.MutatingWebhookConfigurationInformer, + vwcInformer adminformers.ValidatingWebhookConfigurationInformer, resCache resourcecache.ResourceCache, kDeplInformer informers.DeploymentInformer, nsInformer coreinformers.NamespaceInformer, @@ -90,10 +102,14 @@ func NewRegister( createDefaultWebhook: make(chan string), kDeplLister: kDeplInformer.Lister(), kDeplListerSynced: kDeplInformer.Informer().HasSynced, + mwcLister: mwcInformer.Lister(), + vwcLister: vwcInformer.Lister(), + mwcListerSynced: mwcInformer.Informer().HasSynced, + vwcListerSynced: vwcInformer.Informer().HasSynced, stopCh: stopCh, } - register.manage = newWebhookConfigManager(client, kyvernoClient, pInformer, npInformer, resCache, nsInformer, serverIP, register.autoUpdateWebhooks, register.createDefaultWebhook, stopCh, log.WithName("WebhookConfigManager")) + register.manage = newWebhookConfigManager(client, kyvernoClient, pInformer, npInformer, mwcInformer, vwcInformer, resCache, nsInformer, serverIP, register.autoUpdateWebhooks, register.createDefaultWebhook, stopCh, log.WithName("WebhookConfigManager")) return register } @@ -146,33 +162,30 @@ func (wrc *Register) Register() error { } func (wrc *Register) Start() { - if !cache.WaitForCacheSync(wrc.stopCh, wrc.kDeplListerSynced) { + if !cache.WaitForCacheSync(wrc.stopCh, wrc.mwcListerSynced, wrc.vwcListerSynced, wrc.kDeplListerSynced) { wrc.log.Info("failed to sync kyverno deployment informer cache") } } // Check returns an error if any of the webhooks are not configured func (wrc *Register) Check() error { - mutatingCache, _ := wrc.resCache.GetGVRCache(kindMutating) - validatingCache, _ := wrc.resCache.GetGVRCache(kindValidating) - - if _, err := mutatingCache.Lister().Get(wrc.getVerifyWebhookMutatingWebhookName()); err != nil { + if _, err := wrc.mwcLister.Get(wrc.getVerifyWebhookMutatingWebhookName()); err != nil { return err } - if _, err := mutatingCache.Lister().Get(getResourceMutatingWebhookConfigName(wrc.serverIP)); err != nil { + if _, err := wrc.mwcLister.Get(getResourceMutatingWebhookConfigName(wrc.serverIP)); err != nil { return err } - if _, err := validatingCache.Lister().Get(getResourceValidatingWebhookConfigName(wrc.serverIP)); err != nil { + if _, err := wrc.vwcLister.Get(getResourceValidatingWebhookConfigName(wrc.serverIP)); err != nil { return err } - if _, err := mutatingCache.Lister().Get(getPolicyMutatingWebhookConfigurationName(wrc.serverIP)); err != nil { + if _, err := wrc.mwcLister.Get(getPolicyMutatingWebhookConfigurationName(wrc.serverIP)); err != nil { return err } - if _, err := validatingCache.Lister().Get(getPolicyValidatingWebhookConfigurationName(wrc.serverIP)); err != nil { + if _, err := wrc.vwcLister.Get(getPolicyValidatingWebhookConfigurationName(wrc.serverIP)); err != nil { return err } @@ -440,11 +453,9 @@ func (wrc *Register) removePolicyMutatingWebhookConfiguration(wg *sync.WaitGroup logger := wrc.log.WithValues("kind", kindMutating, "name", mutatingConfig) - if mutateCache, ok := wrc.resCache.GetGVRCache("MutatingWebhookConfiguration"); ok { - if _, err := mutateCache.Lister().Get(mutatingConfig); err != nil && errorsapi.IsNotFound(err) { - logger.V(4).Info("webhook not found") - return - } + if _, err := wrc.mwcLister.Get(mutatingConfig); err != nil && errorsapi.IsNotFound(err) { + logger.V(4).Info("webhook not found") + return } err := wrc.client.DeleteResource("", kindMutating, "", mutatingConfig, false) @@ -477,11 +488,9 @@ func (wrc *Register) removePolicyValidatingWebhookConfiguration(wg *sync.WaitGro validatingConfig := getPolicyValidatingWebhookConfigurationName(wrc.serverIP) logger := wrc.log.WithValues("kind", kindValidating, "name", validatingConfig) - if mutateCache, ok := wrc.resCache.GetGVRCache("ValidatingWebhookConfiguration"); ok { - if _, err := mutateCache.Lister().Get(validatingConfig); err != nil && errorsapi.IsNotFound(err) { - logger.V(4).Info("webhook not found") - return - } + if _, err := wrc.vwcLister.Get(validatingConfig); err != nil && errorsapi.IsNotFound(err) { + logger.V(4).Info("webhook not found") + return } logger.V(4).Info("removing validating webhook configuration") @@ -570,11 +579,9 @@ func (wrc *Register) removeVerifyWebhookMutatingWebhookConfig(wg *sync.WaitGroup mutatingConfig := wrc.getVerifyWebhookMutatingWebhookName() logger := wrc.log.WithValues("kind", kindMutating, "name", mutatingConfig) - if mutateCache, ok := wrc.resCache.GetGVRCache("MutatingWebhookConfiguration"); ok { - if _, err := mutateCache.Lister().Get(mutatingConfig); err != nil && errorsapi.IsNotFound(err) { - logger.V(4).Info("webhook not found") - return - } + if _, err := wrc.mwcLister.Get(mutatingConfig); err != nil && errorsapi.IsNotFound(err) { + logger.V(4).Info("webhook not found") + return } err = wrc.client.DeleteResource("", kindMutating, "", mutatingConfig, false) @@ -700,12 +707,21 @@ func getHealthyPodsIP(pods []unstructured.Unstructured) (ips []string, errs []er } func (wrc *Register) updateResourceValidatingWebhookConfiguration(nsSelector map[string]interface{}) error { - validatingCache, _ := wrc.resCache.GetGVRCache(kindValidating) - - resourceValidating, err := validatingCache.Lister().Get(getResourceValidatingWebhookConfigName(wrc.serverIP)) + resourceValidatingTyped, err := wrc.vwcLister.Get(getResourceValidatingWebhookConfigName(wrc.serverIP)) if err != nil { return errors.Wrapf(err, "unable to get validatingWebhookConfigurations") } + resourceValidatingTyped.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "admissionregistration.k8s.io/v1", Kind: kindValidating}) + + resourceValidating := &unstructured.Unstructured{} + rawResc, err := json.Marshal(resourceValidatingTyped) + if err != nil { + return err + } + err = json.Unmarshal(rawResc, &resourceValidating.Object) + if err != nil { + return err + } webhooksUntyped, _, err := unstructured.NestedSlice(resourceValidating.UnstructuredContent(), "webhooks") if err != nil { @@ -741,12 +757,21 @@ func (wrc *Register) updateResourceValidatingWebhookConfiguration(nsSelector map } func (wrc *Register) updateResourceMutatingWebhookConfiguration(nsSelector map[string]interface{}) error { - mutatingCache, _ := wrc.resCache.GetGVRCache(kindMutating) - - resourceMutating, err := mutatingCache.Lister().Get(getResourceMutatingWebhookConfigName(wrc.serverIP)) + resourceMutatingTyped, err := wrc.mwcLister.Get(getResourceMutatingWebhookConfigName(wrc.serverIP)) if err != nil { return errors.Wrapf(err, "unable to get mutatingWebhookConfigurations") } + resourceMutatingTyped.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "admissionregistration.k8s.io/v1", Kind: kindMutating}) + + resourceMutating := &unstructured.Unstructured{} + rawResc, err := json.Marshal(resourceMutatingTyped) + if err != nil { + return err + } + err = json.Unmarshal(rawResc, &resourceMutating.Object) + if err != nil { + return err + } webhooksUntyped, _, err := unstructured.NestedSlice(resourceMutating.UnstructuredContent(), "webhooks") if err != nil { diff --git a/pkg/webhookconfig/resource.go b/pkg/webhookconfig/resource.go index 5dfd75cb5b..5cc72de18d 100644 --- a/pkg/webhookconfig/resource.go +++ b/pkg/webhookconfig/resource.go @@ -102,11 +102,9 @@ func (wrc *Register) removeResourceMutatingWebhookConfiguration(wg *sync.WaitGro configName := getResourceMutatingWebhookConfigName(wrc.serverIP) logger := wrc.log.WithValues("kind", kindMutating, "name", configName) - if mutateCache, ok := wrc.resCache.GetGVRCache("MutatingWebhookConfiguration"); ok { - if _, err := mutateCache.Lister().Get(configName); err != nil && errorsapi.IsNotFound(err) { - logger.V(4).Info("webhook not found") - return - } + if _, err := wrc.mwcLister.Get(configName); err != nil && errorsapi.IsNotFound(err) { + logger.V(4).Info("webhook not found") + return } // delete webhook configuration @@ -204,11 +202,9 @@ func (wrc *Register) removeResourceValidatingWebhookConfiguration(wg *sync.WaitG configName := getResourceValidatingWebhookConfigName(wrc.serverIP) logger := wrc.log.WithValues("kind", kindValidating, "name", configName) - if mutateCache, ok := wrc.resCache.GetGVRCache("ValidatingWebhookConfiguration"); ok { - if _, err := mutateCache.Lister().Get(configName); err != nil && errorsapi.IsNotFound(err) { - logger.V(4).Info("webhook not found") - return - } + if _, err := wrc.vwcLister.Get(configName); err != nil && errorsapi.IsNotFound(err) { + logger.V(4).Info("webhook not found") + return } err := wrc.client.DeleteResource("", kindValidating, "", configName, false)