From a315c22e2f348ee207548b057f64a7cf4bd27460 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Fri, 15 Nov 2019 14:01:40 -0800 Subject: [PATCH] refer informer cache in policy controller for mutatingwebhookconfigs --- main.go | 18 ++++++---- pkg/policy/controller.go | 32 ++++++++++------- pkg/policy/webhookregistration.go | 24 ++++++++++++- pkg/webhookconfig/checker.go | 2 +- pkg/webhookconfig/policy.go | 4 +-- pkg/webhookconfig/registration.go | 58 +++++++++++-------------------- pkg/webhookconfig/resource.go | 18 ++++++---- 7 files changed, 89 insertions(+), 67 deletions(-) diff --git a/main.go b/main.go index 3dae7e248e..90f3153565 100644 --- a/main.go +++ b/main.go @@ -74,10 +74,7 @@ func main() { } // WERBHOOK REGISTRATION CLIENT - webhookRegistrationClient, err := webhookconfig.NewWebhookRegistrationClient(clientConfig, client, serverIP, int32(webhookTimeout)) - if err != nil { - glog.Fatalf("Unable to register admission webhooks on cluster: %v\n", err) - } + webhookRegistrationClient := webhookconfig.NewWebhookRegistrationClient(clientConfig, client, serverIP, int32(webhookTimeout)) // KYVERNO CRD INFORMER // watches CRD resources: @@ -90,7 +87,6 @@ func main() { // watches namespace resource // - cache resync time: 10 seconds kubeInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, 10*time.Second) - // Configuration Data // dyamically load the configuration from configMap // - resource filters @@ -113,7 +109,17 @@ func main() { // - process policy on existing resources // - status aggregator: recieves stats when a policy is applied // & updates the policy status - pc, err := policy.NewPolicyController(pclient, client, pInformer.Kyverno().V1().ClusterPolicies(), pInformer.Kyverno().V1().ClusterPolicyViolations(), pInformer.Kyverno().V1().NamespacedPolicyViolations(), egen, kubeInformer.Admissionregistration().V1beta1().MutatingWebhookConfigurations(), webhookRegistrationClient, configData, pvgen, policyMetaStore) + pc, err := policy.NewPolicyController(pclient, + client, + pInformer.Kyverno().V1().ClusterPolicies(), + pInformer.Kyverno().V1().ClusterPolicyViolations(), + pInformer.Kyverno().V1().NamespacedPolicyViolations(), + kubeInformer.Admissionregistration().V1beta1().MutatingWebhookConfigurations(), + webhookRegistrationClient, + configData, + egen, + pvgen, + policyMetaStore) if err != nil { glog.Fatalf("error creating policy controller: %v\n", err) } diff --git a/pkg/policy/controller.go b/pkg/policy/controller.go index 7f40735750..8fc9cf12e4 100644 --- a/pkg/policy/controller.go +++ b/pkg/policy/controller.go @@ -28,9 +28,9 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" - webhookinformer "k8s.io/client-go/informers/admissionregistration/v1beta1" + mconfiginformer "k8s.io/client-go/informers/admissionregistration/v1beta1" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" - webhooklister "k8s.io/client-go/listers/admissionregistration/v1beta1" + mconfiglister "k8s.io/client-go/listers/admissionregistration/v1beta1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" @@ -71,10 +71,12 @@ type PolicyController struct { pListerSynced cache.InformerSynced // pvListerSynced returns true if the Policy store has been synced at least once pvListerSynced cache.InformerSynced - // pvListerSynced returns true if the Policy store has been synced at least once + // pvListerSynced returns true if the Policy Violation store has been synced at least once nspvListerSynced cache.InformerSynced - // mutationwebhookLister can list/get mutatingwebhookconfigurations - mutationwebhookLister webhooklister.MutatingWebhookConfigurationLister + // mwebhookconfigSynced returns true if the Mutating Webhook Config store has been synced at least once + mwebhookconfigSynced cache.InformerSynced + // list/get mutatingwebhookconfigurations + mWebhookConfigLister mconfiglister.MutatingWebhookConfigurationLister // WebhookRegistrationClient webhookRegistrationClient *webhookconfig.WebhookRegistrationClient // Resource manager, manages the mapping for already processed resource @@ -90,10 +92,15 @@ type PolicyController struct { } // NewPolicyController create a new PolicyController -func NewPolicyController(kyvernoClient *kyvernoclient.Clientset, client *client.Client, pInformer kyvernoinformer.ClusterPolicyInformer, - pvInformer kyvernoinformer.ClusterPolicyViolationInformer, nspvInformer kyvernoinformer.NamespacedPolicyViolationInformer, - eventGen event.Interface, webhookInformer webhookinformer.MutatingWebhookConfigurationInformer, - webhookRegistrationClient *webhookconfig.WebhookRegistrationClient, configHandler config.Interface, +func NewPolicyController(kyvernoClient *kyvernoclient.Clientset, + client *client.Client, + pInformer kyvernoinformer.ClusterPolicyInformer, + pvInformer kyvernoinformer.ClusterPolicyViolationInformer, + nspvInformer kyvernoinformer.NamespacedPolicyViolationInformer, + mconfigwebhookinformer mconfiginformer.MutatingWebhookConfigurationInformer, + webhookRegistrationClient *webhookconfig.WebhookRegistrationClient, + configHandler config.Interface, + eventGen event.Interface, pvGenerator policyviolation.GeneratorInterface, pMetaStore policystore.UpdateInterface) (*PolicyController, error) { // Event broad caster @@ -147,9 +154,8 @@ func NewPolicyController(kyvernoClient *kyvernoclient.Clientset, client *client. pc.pListerSynced = pInformer.Informer().HasSynced pc.pvListerSynced = pvInformer.Informer().HasSynced pc.nspvListerSynced = nspvInformer.Informer().HasSynced - - pc.mutationwebhookLister = webhookInformer.Lister() - + pc.mwebhookconfigSynced = mconfigwebhookinformer.Informer().HasSynced + pc.mWebhookConfigLister = mconfigwebhookinformer.Lister() // resource manager // rebuild after 300 seconds/ 5 mins //TODO: pass the time in seconds instead of converting it internally @@ -394,7 +400,7 @@ func (pc *PolicyController) Run(workers int, stopCh <-chan struct{}) { glog.Info("Starting policy controller") defer glog.Info("Shutting down policy controller") - if !cache.WaitForCacheSync(stopCh, pc.pListerSynced, pc.pvListerSynced, pc.nspvListerSynced) { + if !cache.WaitForCacheSync(stopCh, pc.pListerSynced, pc.pvListerSynced, pc.nspvListerSynced, pc.mwebhookconfigSynced) { return } for i := 0; i < workers; i++ { diff --git a/pkg/policy/webhookregistration.go b/pkg/policy/webhookregistration.go index 45d94994d8..3140fa581c 100644 --- a/pkg/policy/webhookregistration.go +++ b/pkg/policy/webhookregistration.go @@ -9,6 +9,17 @@ import ( func (pc *PolicyController) removeResourceWebhookConfiguration() error { removeWebhookConfig := func() error { var err error + // check informer cache + configName := pc.webhookRegistrationClient.GetResourceMutatingWebhookConfigName() + config, err := pc.mWebhookConfigLister.Get(configName) + if err != nil { + glog.V(4).Infof("failed to list mutating webhook config: %v", err) + return err + } + if config == nil { + // as no resource is found + return nil + } err = pc.webhookRegistrationClient.RemoveResourceMutatingWebhookConfiguration() if err != nil { return err @@ -30,7 +41,7 @@ func (pc *PolicyController) removeResourceWebhookConfiguration() error { return removeWebhookConfig() } - // if there are policies, check if they contain mutating or validating rule + // if polices only have generate rules, we dont need the webhook if !hasMutateOrValidatePolicies(policies) { glog.V(4).Info("no policies with mutating or validating webhook configurations, remove resource webhook configuration if one exists") return removeWebhookConfig() @@ -42,6 +53,17 @@ func (pc *PolicyController) removeResourceWebhookConfiguration() error { func (pc *PolicyController) createResourceMutatingWebhookConfigurationIfRequired(policy kyverno.ClusterPolicy) error { // if the policy contains mutating & validation rules and it config does not exist we create one if policy.HasMutateOrValidate() { + // check cache + configName := pc.webhookRegistrationClient.GetResourceMutatingWebhookConfigName() + config, err := pc.mWebhookConfigLister.Get(configName) + if err != nil { + glog.V(4).Infof("failed to list mutating webhook configuration: %v", err) + return err + } + if config != nil { + // mutating webhoook configuration already exists + return nil + } if err := pc.webhookRegistrationClient.CreateResourceMutatingWebhookConfiguration(); err != nil { return err } diff --git a/pkg/webhookconfig/checker.go b/pkg/webhookconfig/checker.go index 3494fc159f..0c4b3fc656 100644 --- a/pkg/webhookconfig/checker.go +++ b/pkg/webhookconfig/checker.go @@ -67,7 +67,7 @@ func (wrc *WebhookRegistrationClient) removeVerifyWebhookMutatingWebhookConfig() mutatingConfig = config.VerifyMutatingWebhookConfigurationName } glog.V(4).Infof("removing webhook configuration %s", mutatingConfig) - err = wrc.registrationClient.MutatingWebhookConfigurations().Delete(mutatingConfig, &v1.DeleteOptions{}) + err = wrc.client.DeleteResouce(MutatingWebhookConfigurationKind, "", mutatingConfig, false) if errorsapi.IsNotFound(err) { glog.V(4).Infof("verify webhook configuration %s, does not exits. not deleting", mutatingConfig) } else if err != nil { diff --git a/pkg/webhookconfig/policy.go b/pkg/webhookconfig/policy.go index f9075d3ee6..78941fbf0c 100644 --- a/pkg/webhookconfig/policy.go +++ b/pkg/webhookconfig/policy.go @@ -118,7 +118,7 @@ func (wrc *WebhookRegistrationClient) removePolicyWebhookConfigurations() { validatingConfig = config.PolicyValidatingWebhookConfigurationName } glog.V(4).Infof("removing webhook configuration %s", validatingConfig) - err = wrc.registrationClient.ValidatingWebhookConfigurations().Delete(validatingConfig, &v1.DeleteOptions{}) + err = wrc.client.DeleteResouce(ValidatingWebhookConfigurationKind, "", validatingConfig, false) if errorsapi.IsNotFound(err) { glog.V(4).Infof("policy webhook configuration %s, does not exits. not deleting", validatingConfig) } else if err != nil { @@ -136,7 +136,7 @@ func (wrc *WebhookRegistrationClient) removePolicyWebhookConfigurations() { } glog.V(4).Infof("removing webhook configuration %s", mutatingConfig) - err = wrc.registrationClient.MutatingWebhookConfigurations().Delete(mutatingConfig, &v1.DeleteOptions{}) + err = wrc.client.DeleteResouce(MutatingWebhookConfigurationKind, "", mutatingConfig, false) if errorsapi.IsNotFound(err) { glog.V(4).Infof("policy webhook configuration %s, does not exits. not deleting", mutatingConfig) } else if err != nil { diff --git a/pkg/webhookconfig/registration.go b/pkg/webhookconfig/registration.go index 409f9f66a0..e4cb27df0c 100644 --- a/pkg/webhookconfig/registration.go +++ b/pkg/webhookconfig/registration.go @@ -5,41 +5,38 @@ import ( "time" "github.com/golang/glog" - "github.com/nirmata/kyverno/pkg/config" client "github.com/nirmata/kyverno/pkg/dclient" admregapi "k8s.io/api/admissionregistration/v1beta1" errorsapi "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - admregclient "k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1" rest "k8s.io/client-go/rest" ) +const ( + MutatingWebhookConfigurationKind string = "MutatingWebhookConfiguration" + ValidatingWebhookConfigurationKind string = "ValidatingWebhookConfiguration" +) + // WebhookRegistrationClient is client for registration webhooks on cluster type WebhookRegistrationClient struct { - registrationClient *admregclient.AdmissionregistrationV1beta1Client - client *client.Client - clientConfig *rest.Config + client *client.Client + clientConfig *rest.Config // serverIP should be used if running Kyverno out of clutser serverIP string timeoutSeconds int32 } // NewWebhookRegistrationClient creates new WebhookRegistrationClient instance -func NewWebhookRegistrationClient(clientConfig *rest.Config, client *client.Client, serverIP string, webhookTimeout int32) (*WebhookRegistrationClient, error) { - registrationClient, err := admregclient.NewForConfig(clientConfig) - if err != nil { - return nil, err - } - - glog.V(4).Infof("Registering webhook client using serverIP %s\n", serverIP) - +func NewWebhookRegistrationClient( + clientConfig *rest.Config, + client *client.Client, + serverIP string, + webhookTimeout int32) *WebhookRegistrationClient { return &WebhookRegistrationClient{ - registrationClient: registrationClient, - client: client, - clientConfig: clientConfig, - serverIP: serverIP, - timeoutSeconds: webhookTimeout, - }, nil + clientConfig: clientConfig, + client: client, + serverIP: serverIP, + timeoutSeconds: webhookTimeout, + } } // Register creates admission webhooks configs on cluster @@ -105,8 +102,7 @@ func (wrc *WebhookRegistrationClient) CreateResourceMutatingWebhookConfiguration // clientConfig - service config = wrc.constructMutatingWebhookConfig(caData) } - - _, err := wrc.registrationClient.MutatingWebhookConfigurations().Create(config) + _, err := wrc.client.CreateResource(MutatingWebhookConfigurationKind, "", *config, false) if errorsapi.IsAlreadyExists(err) { glog.V(4).Infof("resource mutating webhook configuration %s, already exists. not creating one", config.Name) return nil @@ -118,18 +114,6 @@ func (wrc *WebhookRegistrationClient) CreateResourceMutatingWebhookConfiguration return nil } -//GetResourceMutatingWebhookConfiguration returns the MutatingWebhookConfiguration -func (wrc *WebhookRegistrationClient) GetResourceMutatingWebhookConfiguration() (*admregapi.MutatingWebhookConfiguration, error) { - var name string - if wrc.serverIP != "" { - name = config.MutatingWebhookConfigurationDebugName - } else { - name = config.MutatingWebhookConfigurationName - } - - return wrc.registrationClient.MutatingWebhookConfigurations().Get(name, v1.GetOptions{}) -} - //registerPolicyValidatingWebhookConfiguration create a Validating webhook configuration for Policy CRD func (wrc *WebhookRegistrationClient) createPolicyValidatingWebhookConfiguration() error { var caData []byte @@ -153,7 +137,7 @@ func (wrc *WebhookRegistrationClient) createPolicyValidatingWebhookConfiguration } // create validating webhook configuration resource - if _, err := wrc.registrationClient.ValidatingWebhookConfigurations().Create(config); err != nil { + if _, err := wrc.client.CreateResource(ValidatingWebhookConfigurationKind, "", *config, false); err != nil { return err } @@ -183,7 +167,7 @@ func (wrc *WebhookRegistrationClient) createPolicyMutatingWebhookConfiguration() } // create mutating webhook configuration resource - if _, err := wrc.registrationClient.MutatingWebhookConfigurations().Create(config); err != nil { + if _, err := wrc.client.CreateResource(MutatingWebhookConfigurationKind, "", *config, false); err != nil { return err } @@ -213,7 +197,7 @@ func (wrc *WebhookRegistrationClient) createVerifyMutatingWebhookConfiguration() } // create mutating webhook configuration resource - if _, err := wrc.registrationClient.MutatingWebhookConfigurations().Create(config); err != nil { + if _, err := wrc.client.CreateResource(MutatingWebhookConfigurationKind, "", *config, false); err != nil { return err } diff --git a/pkg/webhookconfig/resource.go b/pkg/webhookconfig/resource.go index ff24d76338..4c3f4f1309 100644 --- a/pkg/webhookconfig/resource.go +++ b/pkg/webhookconfig/resource.go @@ -58,16 +58,20 @@ func (wrc *WebhookRegistrationClient) constructMutatingWebhookConfig(caData []by } } +//GetResourceMutatingWebhookConfigName provi +func (wrc *WebhookRegistrationClient) GetResourceMutatingWebhookConfigName() string { + if wrc.serverIP != "" { + return config.MutatingWebhookConfigurationDebugName + } + return config.MutatingWebhookConfigurationName +} + //RemoveResourceMutatingWebhookConfiguration removes mutating webhook configuration for all resources func (wrc *WebhookRegistrationClient) RemoveResourceMutatingWebhookConfiguration() error { - var configName string - if wrc.serverIP != "" { - configName = config.MutatingWebhookConfigurationDebugName - } else { - configName = config.MutatingWebhookConfigurationName - } + + configName := wrc.GetResourceMutatingWebhookConfigName() // delete webhook configuration - err := wrc.registrationClient.MutatingWebhookConfigurations().Delete(configName, &v1.DeleteOptions{}) + err := wrc.client.DeleteResouce(MutatingWebhookConfigurationKind, "", configName, false) if errors.IsNotFound(err) { glog.V(4).Infof("resource webhook configuration %s does not exits, so not deleting", configName) return nil