diff --git a/main.go b/main.go index 6db33a255e..76e2d79e3f 100644 --- a/main.go +++ b/main.go @@ -13,9 +13,8 @@ import ( "github.com/nirmata/kyverno/pkg/namespace" "github.com/nirmata/kyverno/pkg/policy" "github.com/nirmata/kyverno/pkg/policyviolation" - "github.com/nirmata/kyverno/pkg/utils" "github.com/nirmata/kyverno/pkg/webhooks" - kubeinformers "k8s.io/client-go/informers" + "k8s.io/client-go/informers" "k8s.io/sample-controller/pkg/signals" ) @@ -28,6 +27,8 @@ var ( webhookTimeout int ) +const defaultReSyncTime = 10 * time.Second + func main() { defer glog.Flush() printVersionInfo() @@ -59,40 +60,14 @@ func main() { // - Policy // - PolicyVolation // - cache resync time: 10 seconds - pInformer := kyvernoinformer.NewSharedInformerFactoryWithOptions(pclient, 10*time.Second) + pInformer := kyvernoinformer.NewSharedInformerFactoryWithOptions(pclient, defaultReSyncTime) // EVENT GENERATOR // - generate event with retry egen := event.NewEventGenerator(client, pInformer.Kyverno().V1alpha1().Policies()) - // POLICY CONTROLLER - // - reconciliation policy and policy violation - // - process policy on existing resources - // - status: violation count - - pc, err := policy.NewPolicyController(pclient, client, pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations(), egen) - if err != nil { - glog.Fatalf("error creating policy controller: %v\n", err) - } - - // POLICY VIOLATION CONTROLLER - // status: lastUpdatTime - pvc, err := policyviolation.NewPolicyViolationController(client, pclient, pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations()) - if err != nil { - glog.Fatalf("error creating policy violation controller: %v\n", err) - } - - // NAMESPACE INFORMER - // watches namespace resource - // - cache resync time: 10 seconds - kubeClient, err := utils.NewKubeClient(clientConfig) - if err != nil { - glog.Fatalf("Error creating kubernetes client: %v\n", err) - } - kubeInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, 10*time.Second) - - // GENERATE CONTROLLER - // - watches for Namespace resource and generates resource based on the policy generate rule - nsc := namespace.NewNamespaceController(pclient, client, kubeInformer.Core().V1().Namespaces(), pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations(), egen) + // mutatingWebhookConfiguration Informer + kubeInformer := informers.NewSharedInformerFactory(client.Kclient, defaultReSyncTime) + mutatingWebhookConfigurationLister := kubeInformer.Admissionregistration().V1beta1().MutatingWebhookConfigurations().Lister() tlsPair, err := initTLSPemPair(clientConfig, client) if err != nil { @@ -110,6 +85,28 @@ func main() { if err = webhookRegistrationClient.Register(); err != nil { glog.Fatalf("Failed registering Admission Webhooks: %v\n", err) } + + // POLICY CONTROLLER + // - reconciliation policy and policy violation + // - process policy on existing resources + // - status: violation count + + pc, err := policy.NewPolicyController(pclient, client, pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations(), egen, mutatingWebhookConfigurationLister, webhookRegistrationClient) + if err != nil { + glog.Fatalf("error creating policy controller: %v\n", err) + } + + // POLICY VIOLATION CONTROLLER + // status: lastUpdatTime + pvc, err := policyviolation.NewPolicyViolationController(client, pclient, pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations()) + if err != nil { + glog.Fatalf("error creating policy violation controller: %v\n", err) + } + + // GENERATE CONTROLLER + // - watches for Namespace resource and generates resource based on the policy generate rule + nsc := namespace.NewNamespaceController(pclient, client, kubeInformer.Core().V1().Namespaces(), pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations(), egen) + server, err := webhooks.NewWebhookServer(pclient, client, tlsPair, pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations(), egen, webhookRegistrationClient, filterK8Resources) if err != nil { glog.Fatalf("Unable to create webhook server: %v\n", err) diff --git a/pkg/dclient/client.go b/pkg/dclient/client.go index 50a1f1d3e2..4c46dd13f9 100644 --- a/pkg/dclient/client.go +++ b/pkg/dclient/client.go @@ -31,7 +31,7 @@ type Client struct { client dynamic.Interface cachedClient discovery.CachedDiscoveryInterface clientConfig *rest.Config - kclient kubernetes.Interface + Kclient kubernetes.Interface DiscoveryClient IDiscovery } @@ -48,7 +48,7 @@ func NewClient(config *rest.Config) (*Client, error) { client := Client{ client: dclient, clientConfig: config, - kclient: kclient, + Kclient: kclient, } // Set discovery client // @@ -75,12 +75,12 @@ func (c *Client) GetKubePolicyDeployment() (*apps.Deployment, error) { //TODO: can we use dynamic client to fetch the typed interface // or generate a kube client value to access the interface func (c *Client) GetEventsInterface() (event.EventInterface, error) { - return c.kclient.CoreV1().Events(""), nil + return c.Kclient.CoreV1().Events(""), nil } //GetCSRInterface provides type interface for CSR func (c *Client) GetCSRInterface() (csrtype.CertificateSigningRequestInterface, error) { - return c.kclient.CertificatesV1beta1().CertificateSigningRequests(), nil + return c.Kclient.CertificatesV1beta1().CertificateSigningRequests(), nil } func (c *Client) getInterface(resource string) dynamic.NamespaceableResourceInterface { diff --git a/pkg/dclient/client_test.go b/pkg/dclient/client_test.go index db4396f1c0..2673698b18 100644 --- a/pkg/dclient/client_test.go +++ b/pkg/dclient/client_test.go @@ -3,7 +3,7 @@ package client import ( "testing" - policytypes "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" + policytypes "github.com/nirmata/kyverno/pkg/api/kyverno/v1alpha1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -130,7 +130,7 @@ func TestGenerateResource(t *testing.T) { } gen := policytypes.Generation{Kind: "TheKind", Name: "gen-kind", - Clone: &policytypes.CloneFrom{Namespace: "ns-foo", Name: "name-foo"}} + Clone: policytypes.CloneFrom{Namespace: "ns-foo", Name: "name-foo"}} err = f.client.GenerateResource(gen, ns.GetName(), false) if err != nil { t.Errorf("GenerateResource not working: %s", err) diff --git a/pkg/dclient/utils.go b/pkg/dclient/utils.go index 6492187038..d82500f0d0 100644 --- a/pkg/dclient/utils.go +++ b/pkg/dclient/utils.go @@ -32,7 +32,7 @@ func NewMockClient(scheme *runtime.Scheme, objects ...runtime.Object) (*Client, kclient := kubernetesfake.NewSimpleClientset(objects...) return &Client{ client: client, - kclient: kclient, + Kclient: kclient, }, nil } diff --git a/pkg/policy/controller.go b/pkg/policy/controller.go index 142f28594f..7438c4f2c5 100644 --- a/pkg/policy/controller.go +++ b/pkg/policy/controller.go @@ -13,9 +13,11 @@ import ( "github.com/nirmata/kyverno/pkg/client/clientset/versioned/scheme" kyvernoinformer "github.com/nirmata/kyverno/pkg/client/informers/externalversions/kyverno/v1alpha1" kyvernolister "github.com/nirmata/kyverno/pkg/client/listers/kyverno/v1alpha1" + "github.com/nirmata/kyverno/pkg/config" client "github.com/nirmata/kyverno/pkg/dclient" "github.com/nirmata/kyverno/pkg/event" "github.com/nirmata/kyverno/pkg/utils" + "github.com/nirmata/kyverno/pkg/webhooks" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,6 +28,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + v1beta1 "k8s.io/client-go/listers/admissionregistration/v1beta1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" @@ -62,8 +65,12 @@ type PolicyController struct { pvLister kyvernolister.PolicyViolationLister // pListerSynced returns true if the Policy store has been synced at least once pListerSynced cache.InformerSynced - // pvListerSynced retrns true if the Policy store has been synced at least once + // pvListerSynced returns true if the Policy store has been synced at least once pvListerSynced cache.InformerSynced + // mutationwebhookInformer can list/get mutatingwebhookconfigurations + mutationwebhookInformer v1beta1.MutatingWebhookConfigurationLister + // WebhookRegistrationClient + webhookRegistrationClient *webhooks.WebhookRegistrationClient // Resource manager, manages the mapping for already processed resource rm resourceManager // filter the resources defined in the list @@ -71,7 +78,8 @@ type PolicyController struct { } // NewPolicyController create a new PolicyController -func NewPolicyController(kyvernoClient *kyvernoclient.Clientset, client *client.Client, pInformer kyvernoinformer.PolicyInformer, pvInformer kyvernoinformer.PolicyViolationInformer, eventGen event.Interface) (*PolicyController, error) { +func NewPolicyController(kyvernoClient *kyvernoclient.Clientset, client *client.Client, pInformer kyvernoinformer.PolicyInformer, pvInformer kyvernoinformer.PolicyViolationInformer, + eventGen event.Interface, mutationwebhookInformer v1beta1.MutatingWebhookConfigurationLister, webhookRegistrationClient *webhooks.WebhookRegistrationClient) (*PolicyController, error) { // Event broad caster eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartLogging(glog.Infof) @@ -82,11 +90,13 @@ func NewPolicyController(kyvernoClient *kyvernoclient.Clientset, client *client. eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: eventInterface}) pc := PolicyController{ - client: client, - kyvernoClient: kyvernoClient, - eventGen: eventGen, - eventRecorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "policy_controller"}), - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "policy"), + client: client, + kyvernoClient: kyvernoClient, + eventGen: eventGen, + eventRecorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "policy_controller"}), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "policy"), + mutationwebhookInformer: mutationwebhookInformer, + webhookRegistrationClient: webhookRegistrationClient, } pc.pvControl = RealPVControl{Client: kyvernoClient, Recorder: pc.eventRecorder} @@ -384,13 +394,18 @@ func (pc *PolicyController) syncPolicy(key string) error { policy, err := pc.pLister.Get(key) if errors.IsNotFound(err) { glog.V(2).Infof("Policy %v has been deleted", key) - return nil + err = pc.handleWebhookRegistration(true) + return err } if err != nil { return err } + if err := pc.handleWebhookRegistration(false); err != nil { + glog.Errorln(err) + } + // Deep-copy otherwise we are mutating our cache. // TODO: Deep-copy only when needed. p := policy.DeepCopy() @@ -406,6 +421,33 @@ func (pc *PolicyController) syncPolicy(key string) error { return pc.syncStatusOnly(p, pvList) } +func (pc *PolicyController) handleWebhookRegistration(emptyPolicy bool) error { + selector := &metav1.LabelSelector{MatchLabels: config.KubePolicyAppLabels} + webhookSelector, err := metav1.LabelSelectorAsSelector(selector) + if err != nil { + return fmt.Errorf("invalid label selector: %v", err) + } + + list, err := pc.mutationwebhookInformer.List(webhookSelector) + if err != nil { + return fmt.Errorf("failed to list mutatingwebhookconfigurations, err %v", err) + } + + if emptyPolicy { + // deregister webhookconfigurations it it exists + if list != nil { + pc.webhookRegistrationClient.DeregisterMutatingWebhook() + } + return nil + } + + if list == nil { + pc.webhookRegistrationClient.RegisterMutatingWebhook() + } + + return nil +} + //syncStatusOnly updates the policy status subresource // status: // - violations : (count of the resources that violate this policy ) diff --git a/pkg/webhooks/registration.go b/pkg/webhooks/registration.go index 99c2638148..bbdbf94afb 100644 --- a/pkg/webhooks/registration.go +++ b/pkg/webhooks/registration.go @@ -108,7 +108,7 @@ func (wrc *WebhookRegistrationClient) RegisterPolicyValidatingWebhook() error { // This function does not fail on error: // Register will fail if the config exists, so there is no need to fail on error func (wrc *WebhookRegistrationClient) DeregisterAll() { - wrc.deregisterMutatingWebhook() + wrc.DeregisterMutatingWebhook() wrc.deregisterValidatingWebhook() if wrc.serverIP != "" { @@ -124,11 +124,11 @@ func (wrc *WebhookRegistrationClient) DeregisterAll() { } func (wrc *WebhookRegistrationClient) deregister() { - wrc.deregisterMutatingWebhook() + wrc.DeregisterMutatingWebhook() wrc.deregisterValidatingWebhook() } -func (wrc *WebhookRegistrationClient) deregisterMutatingWebhook() { +func (wrc *WebhookRegistrationClient) DeregisterMutatingWebhook() { if wrc.serverIP != "" { err := wrc.registrationClient.MutatingWebhookConfigurations().Delete(config.MutatingWebhookConfigurationDebug, &v1.DeleteOptions{}) if err != nil && !errorsapi.IsNotFound(err) { diff --git a/pkg/webhooks/webhookManager.go b/pkg/webhooks/webhookManager.go index 2cb082a91d..5a498f68fa 100644 --- a/pkg/webhooks/webhookManager.go +++ b/pkg/webhooks/webhookManager.go @@ -41,7 +41,7 @@ func (ws *WebhookServer) deregisterWebhookConfigurations(policy kyverno.Policy) // deregister webhook if no policy found in cluster if len(policies) == 1 { - ws.webhookRegistrationClient.deregisterMutatingWebhook() + ws.webhookRegistrationClient.DeregisterMutatingWebhook() glog.Infoln("Mutating webhook deregistered") }