diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index 988dced02a..e377434c1e 100755 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -9,7 +9,6 @@ import ( "os" "time" - "github.com/kyverno/kyverno/pkg/checker" kyvernoclient "github.com/kyverno/kyverno/pkg/client/clientset/versioned" kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions" "github.com/kyverno/kyverno/pkg/common" @@ -137,7 +136,7 @@ func main() { kubeInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, resyncPeriod) kubedynamicInformer := client.NewDynamicSharedInformerFactory(resyncPeriod) - webhookRegistrationClient := webhookconfig.NewWebhookRegistrationClient( + webhookCfg := webhookconfig.NewRegister( clientConfig, client, serverIP, @@ -145,15 +144,8 @@ func main() { log.Log) // Resource Mutating Webhook Watcher - lastReqTime := checker.NewLastReqTime(log.Log.WithName("LastReqTime")) - rWebhookWatcher := webhookconfig.NewResourceWebhookRegister( - lastReqTime, - kubeInformer.Admissionregistration().V1beta1().MutatingWebhookConfigurations(), - kubeInformer.Admissionregistration().V1beta1().ValidatingWebhookConfigurations(), - webhookRegistrationClient, - runValidationInMutatingWebhook, - log.Log.WithName("ResourceWebhookRegister"), - ) + webhookMonitor := webhookconfig.NewMonitor(log.Log.WithName("WebhookMonitor")) + // KYVERNO CRD INFORMER // watches CRD resources: @@ -224,7 +216,6 @@ func main() { configData, eventGenerator, reportReqGen, - rWebhookWatcher, kubeInformer.Core().V1().Namespaces(), log.Log.WithName("PolicyController"), rCache, @@ -282,20 +273,16 @@ func main() { rCache, ) - // CONFIGURE CERTIFICATES + // Configure certificates tlsPair, err := client.InitTLSPemPair(clientConfig, fqdncn) if err != nil { setupLog.Error(err, "Failed to initialize TLS key/certificate pair") os.Exit(1) } - // WEBHOOK REGISTRATION - // - mutating,validatingwebhookconfiguration (Policy) - // - verifymutatingwebhookconfiguration (Kyverno Deployment) - // resource webhook confgiuration is generated dynamically in the webhook server and policy controller - // based on the policy resources created - if err = webhookRegistrationClient.Register(); err != nil { - setupLog.Error(err, "Failed to register Admission webhooks") + // Register webhookCfg + if err = webhookCfg.Register(); err != nil { + setupLog.Error(err, "Failed to register admission control webhooks") os.Exit(1) } @@ -327,12 +314,12 @@ func main() { kubeInformer.Rbac().V1().ClusterRoles(), eventGenerator, pCacheController.Cache, - webhookRegistrationClient, + webhookCfg, + webhookMonitor, statusSync.Listener, configData, reportReqGen, grgen, - rWebhookWatcher, auditHandler, supportMutateValidate, cleanUp, @@ -354,7 +341,6 @@ func main() { go reportReqGen.Run(2, stopCh) go prgen.Run(1, stopCh) go grgen.Run(1) - go rWebhookWatcher.Run(stopCh) go configData.Run(stopCh) go policyCtrl.Run(2, stopCh) go eventGenerator.Run(3, stopCh) @@ -366,11 +352,7 @@ func main() { openAPISync.Run(1, stopCh) // verifies if the admission control is enabled and active - // resync: 60 seconds - // deadline: 60 seconds (send request) - // max deadline: deadline*3 (set the deployment annotation as false) server.RunAsync(stopCh) - <-stopCh // by default http.Server waits indefinitely for connections to return to idle and then shuts down diff --git a/pkg/checker/checker.go b/pkg/checker/checker.go deleted file mode 100644 index 0ab9c4dbc6..0000000000 --- a/pkg/checker/checker.go +++ /dev/null @@ -1,129 +0,0 @@ -package checker - -import ( - "fmt" - "sync" - "time" - - "github.com/go-logr/logr" - kyvernolister "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1" - dclient "github.com/kyverno/kyverno/pkg/dclient" - "github.com/kyverno/kyverno/pkg/event" - "k8s.io/apimachinery/pkg/labels" -) - -//MaxRetryCount defines the max deadline count -const ( - MaxRetryCount int = 3 - DefaultDeadline time.Duration = 60 * time.Second - DefaultResync time.Duration = 60 * time.Second -) - -// LastReqTime stores the lastrequest times for incoming api-requests -type LastReqTime struct { - t time.Time - mu sync.RWMutex - log logr.Logger -} - -//Time returns the lastrequest time -func (t *LastReqTime) Time() time.Time { - t.mu.RLock() - defer t.mu.RUnlock() - return t.t -} - -//SetTime updates the lastrequest time -func (t *LastReqTime) SetTime(tm time.Time) { - t.mu.Lock() - defer t.mu.Unlock() - - t.t = tm -} - -//NewLastReqTime returns a new instance of LastRequestTime store -func NewLastReqTime(log logr.Logger) *LastReqTime { - return &LastReqTime{ - t: time.Now(), - log: log, - } -} - -func checkIfPolicyWithMutateAndGenerateExists(pLister kyvernolister.ClusterPolicyLister, log logr.Logger) bool { - policies, err := pLister.ListResources(labels.NewSelector()) - if err != nil { - log.Error(err, "failed to list cluster policies") - } - - for _, policy := range policies { - if policy.HasMutateOrValidateOrGenerate() { - // as there exists one policy with mutate or validate rule - // so there must be a webhook configuration on resource - return true - } - } - - return false -} - -//Run runs the checker and verify the resource update -func (t *LastReqTime) Run(pLister kyvernolister.ClusterPolicyLister, eventGen event.Interface, client *dclient.Client, defaultResync time.Duration, deadline time.Duration, stopCh <-chan struct{}) { - logger := t.log - logger.V(4).Info("starting default resync for webhook checker", "resyncTime", defaultResync) - maxDeadline := deadline * time.Duration(MaxRetryCount) - ticker := time.NewTicker(defaultResync) - /// interface to update and increment kyverno webhook status via annotations - statuscontrol := NewVerifyControl(client, eventGen, logger.WithName("StatusControl")) - // send the initial update status - if checkIfPolicyWithMutateAndGenerateExists(pLister, logger) { - if err := statuscontrol.SuccessStatus(); err != nil { - logger.Error(err, "failed to set 'success' status") - } - } - - defer ticker.Stop() - // - has received request -> set webhookstatus as "True" - // - no requests received - // -> if greater than deadline, send update request - // -> if greater than maxDeadline, send failed status update - for { - select { - case <-ticker.C: - if !checkIfPolicyWithMutateAndGenerateExists(pLister, logger) { - continue - } - - timeDiff := time.Since(t.Time()) - if timeDiff > maxDeadline { - err := fmt.Errorf("admission control configuration error") - logger.Error(err, "webhook check failed", "deadline", maxDeadline) - if err := statuscontrol.FailedStatus(); err != nil { - logger.Error(err, "error setting webhook check status to failed") - } - - continue - } - - if timeDiff > deadline { - logger.V(3).Info("webhook check deadline exceeded", "deadline", deadline) - // send request to update the kyverno deployment - if err := statuscontrol.IncrementAnnotation(); err != nil { - logger.Error(err, "failed to increment annotation") - } - - continue - } - - // if the status was false before then we update it to true - // send request to update the kyverno deployment - if err := statuscontrol.SuccessStatus(); err != nil { - logger.Error(err, "error setting webhook check status to success") - } - - case <-stopCh: - // handler termination signal - logger.V(2).Info("stopping default resync for webhook checker") - return - } - } -} diff --git a/pkg/config/config.go b/pkg/config/config.go index 03f635ac07..77da1f2eff 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -15,32 +15,32 @@ const ( //MutatingWebhookConfigurationDebugName default resource mutating webhook configuration name for debug mode MutatingWebhookConfigurationDebugName = "kyverno-resource-mutating-webhook-cfg-debug" //MutatingWebhookName default resource mutating webhook name - MutatingWebhookName = "nirmata.kyverno.resource.mutating-webhook" + MutatingWebhookName = "mutate.kyverno.svc" ValidatingWebhookConfigurationName = "kyverno-resource-validating-webhook-cfg" ValidatingWebhookConfigurationDebugName = "kyverno-resource-validating-webhook-cfg-debug" - ValidatingWebhookName = "nirmata.kyverno.resource.validating-webhook" + ValidatingWebhookName = "validate.kyverno.svc" //VerifyMutatingWebhookConfigurationName default verify mutating webhook configuration name VerifyMutatingWebhookConfigurationName = "kyverno-verify-mutating-webhook-cfg" //VerifyMutatingWebhookConfigurationDebugName default verify mutating webhook configuration name for debug mode VerifyMutatingWebhookConfigurationDebugName = "kyverno-verify-mutating-webhook-cfg-debug" //VerifyMutatingWebhookName default verify mutating webhook name - VerifyMutatingWebhookName = "nirmata.kyverno.verify-mutating-webhook" + VerifyMutatingWebhookName = "monitor-webhooks.kyverno.svc" //PolicyValidatingWebhookConfigurationName default policy validating webhook configuration name PolicyValidatingWebhookConfigurationName = "kyverno-policy-validating-webhook-cfg" //PolicyValidatingWebhookConfigurationDebugName default policy validating webhook configuration name for debug mode PolicyValidatingWebhookConfigurationDebugName = "kyverno-policy-validating-webhook-cfg-debug" //PolicyValidatingWebhookName default policy validating webhook name - PolicyValidatingWebhookName = "nirmata.kyverno.policy-validating-webhook" + PolicyValidatingWebhookName = "validate-policy.kyverno.svc" //PolicyMutatingWebhookConfigurationName default policy mutating webhook configuration name PolicyMutatingWebhookConfigurationName = "kyverno-policy-mutating-webhook-cfg" //PolicyMutatingWebhookConfigurationDebugName default policy mutating webhook configuration name for debug mode PolicyMutatingWebhookConfigurationDebugName = "kyverno-policy-mutating-webhook-cfg-debug" //PolicyMutatingWebhookName default policy mutating webhook name - PolicyMutatingWebhookName = "nirmata.kyverno.policy-mutating-webhook" + PolicyMutatingWebhookName = "mutate-policy.kyverno.svc" // Due to kubernetes issue, we must use next literal constants instead of deployment TypeMeta fields // Issue: https://github.com/kubernetes/kubernetes/pull/63972 @@ -54,26 +54,33 @@ const ( ) var ( - //KubePolicyNamespace is the kyverno policy namespace - KubePolicyNamespace = getKyvernoNameSpace() - // KubePolicyDeploymentName define the default deployment namespace - KubePolicyDeploymentName = "kyverno" + //KyvernoNamespace is the Kyverno namespace + KyvernoNamespace = getKyvernoNameSpace() - //WebhookServiceName default kyverno webhook service name - WebhookServiceName = getWebhookServiceName() + // KyvernoDeploymentName is the Kyverno deployment name + KyvernoDeploymentName = getKyvernoDeploymentName() + + //KyvernoServiceName is the Kyverno service name + KyvernoServiceName = getKyvernoServiceName() //MutatingWebhookServicePath is the path for mutation webhook MutatingWebhookServicePath = "/mutate" + //ValidatingWebhookServicePath is the path for validation webhook ValidatingWebhookServicePath = "/validate" + //PolicyValidatingWebhookServicePath is the path for policy validation webhook(used to validate policy resource) PolicyValidatingWebhookServicePath = "/policyvalidate" + //PolicyMutatingWebhookServicePath is the path for policy mutation webhook(used to default) PolicyMutatingWebhookServicePath = "/policymutate" + //VerifyMutatingWebhookServicePath is the path for verify webhook(used to veryfing if admission control is enabled and active) VerifyMutatingWebhookServicePath = "/verifymutate" + // LivenessServicePath is the path for check liveness health LivenessServicePath = "/health/liveness" + // ReadinessServicePath is the path for check readness health ReadinessServicePath = "/health/readiness" ) @@ -98,11 +105,20 @@ func getKyvernoNameSpace() string { return kyvernoNamespace } -// getWebhookServiceName - setting default WebhookServiceName -func getWebhookServiceName() string { +// getKyvernoServiceName - setting default KyvernoServiceName +func getKyvernoServiceName() string { webhookServiceName := os.Getenv("KYVERNO_SVC") if webhookServiceName == "" { webhookServiceName = "kyverno-svc" } return webhookServiceName } + +// getKyvernoDeploymentName - setting default KyvernoServiceName +func getKyvernoDeploymentName() string { + name := os.Getenv("KYVERNO_DEPLOYMENT") + if name == "" { + name = "kyverno" + } + return name +} diff --git a/pkg/dclient/certificates.go b/pkg/dclient/certificates.go index 00112b2c8b..e16d36224b 100644 --- a/pkg/dclient/certificates.go +++ b/pkg/dclient/certificates.go @@ -234,8 +234,8 @@ func (c *Client) GetTLSCertProps(configuration *rest.Config) (certProps tls.Cert return certProps, err } certProps = tls.CertificateProps{ - Service: config.WebhookServiceName, - Namespace: config.KubePolicyNamespace, + Service: config.KyvernoServiceName, + Namespace: config.KyvernoNamespace, APIServerHost: apiServerURL.Hostname(), } return certProps, nil diff --git a/pkg/dclient/client.go b/pkg/dclient/client.go index ff77acc0b0..817e3e7e10 100644 --- a/pkg/dclient/client.go +++ b/pkg/dclient/client.go @@ -75,7 +75,7 @@ func (c *Client) NewDynamicSharedInformerFactory(defaultResync time.Duration) dy //GetKubePolicyDeployment returns kube policy depoyment value func (c *Client) GetKubePolicyDeployment() (*apps.Deployment, error) { - kubePolicyDeployment, err := c.GetResource("", "Deployment", config.KubePolicyNamespace, config.KubePolicyDeploymentName) + kubePolicyDeployment, err := c.GetResource("", "Deployment", config.KyvernoNamespace, config.KyvernoDeploymentName) if err != nil { return nil, err } diff --git a/pkg/dclient/client_test.go b/pkg/dclient/client_test.go index 34247c70de..9689d1d8af 100644 --- a/pkg/dclient/client_test.go +++ b/pkg/dclient/client_test.go @@ -40,7 +40,7 @@ func newFixture(t *testing.T) *fixture { newUnstructured("group/version", "TheKind", "ns-foo", "name-bar"), newUnstructured("group/version", "TheKind", "ns-foo", "name-baz"), newUnstructured("group2/version", "TheKind", "ns-foo", "name2-baz"), - newUnstructured("apps/v1", "Deployment", config.KubePolicyNamespace, config.KubePolicyDeploymentName), + newUnstructured("apps/v1", "Deployment", config.KyvernoNamespace, config.KyvernoDeploymentName), } scheme := runtime.NewScheme() // Create mock client diff --git a/pkg/generate/cleanup/controller.go b/pkg/generate/cleanup/controller.go index 46f94817da..fe645bdc77 100644 --- a/pkg/generate/cleanup/controller.go +++ b/pkg/generate/cleanup/controller.go @@ -80,7 +80,7 @@ func NewController( c.syncHandler = c.syncGenerateRequest c.pLister = pInformer.Lister() - c.grLister = grInformer.Lister().GenerateRequests(config.KubePolicyNamespace) + c.grLister = grInformer.Lister().GenerateRequests(config.KyvernoNamespace) c.pSynced = pInformer.Informer().HasSynced c.grSynced = grInformer.Informer().HasSynced diff --git a/pkg/generate/cleanup/resource.go b/pkg/generate/cleanup/resource.go index 1ca2d01f48..cade7410a1 100644 --- a/pkg/generate/cleanup/resource.go +++ b/pkg/generate/cleanup/resource.go @@ -20,5 +20,5 @@ type Control struct { //Delete deletes the specified resource func (c Control) Delete(gr string) error { - return c.client.KyvernoV1().GenerateRequests(config.KubePolicyNamespace).Delete(context.TODO(), gr, metav1.DeleteOptions{}) + return c.client.KyvernoV1().GenerateRequests(config.KyvernoNamespace).Delete(context.TODO(), gr, metav1.DeleteOptions{}) } diff --git a/pkg/generate/controller.go b/pkg/generate/controller.go index 8bd541c5b1..a11d258b00 100644 --- a/pkg/generate/controller.go +++ b/pkg/generate/controller.go @@ -109,7 +109,7 @@ func NewController( c.syncHandler = c.syncGenerateRequest c.pLister = pInformer.Lister() - c.grLister = grInformer.Lister().GenerateRequests(config.KubePolicyNamespace) + c.grLister = grInformer.Lister().GenerateRequests(config.KyvernoNamespace) c.pSynced = pInformer.Informer().HasSynced c.grSynced = pInformer.Informer().HasSynced diff --git a/pkg/generate/generate.go b/pkg/generate/generate.go index 30964857b2..632a1dbbd6 100644 --- a/pkg/generate/generate.go +++ b/pkg/generate/generate.go @@ -122,7 +122,7 @@ func (c *Controller) applyGenerate(resource unstructured.Unstructured, gr kyvern // Removing GR if rule is failed. Used when the generate condition failed but gr exist for _, r := range engineResponse.PolicyResponse.Rules { if !r.Success { - grList, err := c.kyvernoClient.KyvernoV1().GenerateRequests(config.KubePolicyNamespace).List(contextdefault.TODO(), metav1.ListOptions{}) + grList, err := c.kyvernoClient.KyvernoV1().GenerateRequests(config.KyvernoNamespace).List(contextdefault.TODO(), metav1.ListOptions{}) if err != nil { logger.Error(err, "failed to list generate requests") continue @@ -130,7 +130,7 @@ func (c *Controller) applyGenerate(resource unstructured.Unstructured, gr kyvern for _, v := range grList.Items { if engineResponse.PolicyResponse.Policy == v.Spec.Policy && engineResponse.PolicyResponse.Resource.Name == v.Spec.Resource.Name && engineResponse.PolicyResponse.Resource.Kind == v.Spec.Resource.Kind && engineResponse.PolicyResponse.Resource.Namespace == v.Spec.Resource.Namespace { - err := c.kyvernoClient.KyvernoV1().GenerateRequests(config.KubePolicyNamespace).Delete(contextdefault.TODO(), v.GetName(), metav1.DeleteOptions{}) + err := c.kyvernoClient.KyvernoV1().GenerateRequests(config.KyvernoNamespace).Delete(contextdefault.TODO(), v.GetName(), metav1.DeleteOptions{}) if err != nil { logger.Error(err, " failed to delete generate request") } diff --git a/pkg/generate/status.go b/pkg/generate/status.go index 5a5bb2d4bd..7c204f33dc 100644 --- a/pkg/generate/status.go +++ b/pkg/generate/status.go @@ -28,7 +28,7 @@ func (sc StatusControl) Failed(gr kyverno.GenerateRequest, message string, genRe gr.Status.Message = message // Update Generated Resources gr.Status.GeneratedResources = genResources - _, err := sc.client.KyvernoV1().GenerateRequests(config.KubePolicyNamespace).UpdateStatus(context.TODO(), &gr, v1.UpdateOptions{}) + _, err := sc.client.KyvernoV1().GenerateRequests(config.KyvernoNamespace).UpdateStatus(context.TODO(), &gr, v1.UpdateOptions{}) if err != nil && !errors.IsNotFound(err) { log.Log.Error(err, "failed to update generate request status", "name", gr.Name) return err @@ -44,7 +44,7 @@ func (sc StatusControl) Success(gr kyverno.GenerateRequest, genResources []kyver // Update Generated Resources gr.Status.GeneratedResources = genResources - _, err := sc.client.KyvernoV1().GenerateRequests(config.KubePolicyNamespace).UpdateStatus(context.TODO(), &gr, v1.UpdateOptions{}) + _, err := sc.client.KyvernoV1().GenerateRequests(config.KyvernoNamespace).UpdateStatus(context.TODO(), &gr, v1.UpdateOptions{}) if err != nil && !errors.IsNotFound(err) { log.Log.Error(err, "failed to update generate request status", "name", gr.Name) return err diff --git a/pkg/policy/controller.go b/pkg/policy/controller.go index afb729a213..0aaae912b9 100644 --- a/pkg/policy/controller.go +++ b/pkg/policy/controller.go @@ -21,7 +21,6 @@ import ( "github.com/kyverno/kyverno/pkg/event" "github.com/kyverno/kyverno/pkg/policyreport" "github.com/kyverno/kyverno/pkg/resourcecache" - "github.com/kyverno/kyverno/pkg/webhookconfig" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -94,9 +93,6 @@ type PolicyController struct { // policy report generator prGenerator policyreport.GeneratorInterface - // resourceWebhookWatcher queues the webhook creation request, creates the webhook - resourceWebhookWatcher *webhookconfig.ResourceWebhookRegister - // resCache - controls creation and fetching of resource informer cache resCache resourcecache.ResourceCacheIface @@ -112,7 +108,6 @@ func NewPolicyController(kyvernoClient *kyvernoclient.Clientset, configHandler config.Interface, eventGen event.Interface, prGenerator policyreport.GeneratorInterface, - resourceWebhookWatcher *webhookconfig.ResourceWebhookRegister, namespaces informers.NamespaceInformer, log logr.Logger, resCache resourcecache.ResourceCacheIface) (*PolicyController, error) { @@ -134,7 +129,6 @@ func NewPolicyController(kyvernoClient *kyvernoclient.Clientset, queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "policy"), configHandler: configHandler, prGenerator: prGenerator, - resourceWebhookWatcher: resourceWebhookWatcher, log: log, resCache: resCache, } @@ -352,11 +346,6 @@ func (pc *PolicyController) worker() { } func (pc *PolicyController) processNextWorkItem() bool { - // if policies exist before Kyverno get created, resource webhook configuration - // could not be registered as clusterpolicy.spec.background=false by default - // the policy controller would starts only when the first incoming policy is queued - pc.resourceWebhookWatcher.RegisterResourceWebhook() - key, quit := pc.queue.Get() if quit { return false @@ -410,15 +399,10 @@ func (pc *PolicyController) syncPolicy(key string) error { } if err != nil { - // remove webhook configurations if there are no policies - if err := pc.removeResourceWebhookConfiguration(); err != nil { - logger.Error(err, "failed to remove resource webhook configurations") - } - if errors.IsNotFound(err) { for _, v := range grList { if key == v.Spec.Policy { - err := pc.kyvernoClient.KyvernoV1().GenerateRequests(config.KubePolicyNamespace).Delete(context.TODO(), v.GetName(), metav1.DeleteOptions{}) + err := pc.kyvernoClient.KyvernoV1().GenerateRequests(config.KyvernoNamespace).Delete(context.TODO(), v.GetName(), metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { logger.Error(err, "failed to delete gr") } @@ -428,6 +412,7 @@ func (pc *PolicyController) syncPolicy(key string) error { go pc.removeResultsEntryFromPolicyReport(key) return nil } + return err } @@ -436,14 +421,13 @@ func (pc *PolicyController) syncPolicy(key string) error { v.SetLabels(map[string]string{ "policy-update": fmt.Sprintf("revision-count-%d", rand.Intn(100000)), }) - _, err := pc.kyvernoClient.KyvernoV1().GenerateRequests(config.KubePolicyNamespace).Update(context.TODO(), v, metav1.UpdateOptions{}) + _, err := pc.kyvernoClient.KyvernoV1().GenerateRequests(config.KyvernoNamespace).Update(context.TODO(), v, metav1.UpdateOptions{}) if err != nil { logger.Error(err, "failed to update gr", "policy", policy.GetName(), "gr", v.GetName()) } } } - pc.resourceWebhookWatcher.RegisterResourceWebhook() engineResponses := pc.processExistingResources(policy) pc.cleanupAndReport(engineResponses) return nil diff --git a/pkg/policy/webhookregistration.go b/pkg/policy/webhookregistration.go deleted file mode 100644 index 33e7ee0795..0000000000 --- a/pkg/policy/webhookregistration.go +++ /dev/null @@ -1,23 +0,0 @@ -package policy - -import ( - "k8s.io/apimachinery/pkg/labels" -) - -func (pc *PolicyController) removeResourceWebhookConfiguration() error { - logger := pc.log - var err error - // get all existing policies - policies, err := pc.pLister.List(labels.NewSelector()) - if err != nil { - logger.Error(err, "failed to list policies") - return err - } - - if len(policies) == 0 { - logger.V(4).Info("no policies loaded, removing resource webhook configuration if one exists") - pc.resourceWebhookWatcher.RemoveResourceWebhookConfiguration() - } - - return nil -} diff --git a/pkg/policyreport/builder.go b/pkg/policyreport/builder.go index 30954ade68..1e835fa7fb 100755 --- a/pkg/policyreport/builder.go +++ b/pkg/policyreport/builder.go @@ -172,7 +172,7 @@ func (builder *requestBuilder) build(info Info) (req *unstructured.Unstructured, func set(obj *unstructured.Unstructured, info Info) { resource := info.Resource - obj.SetNamespace(config.KubePolicyNamespace) + obj.SetNamespace(config.KyvernoNamespace) obj.SetAPIVersion(request.SchemeGroupVersion.Group + "/" + request.SchemeGroupVersion.Version) if resource.GetNamespace() == "" { obj.SetGenerateName(clusterreportchangerequest + "-") diff --git a/pkg/policyreport/reportcontroller.go b/pkg/policyreport/reportcontroller.go index 963fbafda1..2104716e10 100644 --- a/pkg/policyreport/reportcontroller.go +++ b/pkg/policyreport/reportcontroller.go @@ -369,7 +369,7 @@ func (g *ReportGenerator) removePolicyEntryFromReport(policyName, ruleName strin deletedLabelRule: ruleName, }) } - aggregatedRequests, err := g.reportChangeRequestLister.ReportChangeRequests(config.KubePolicyNamespace).List(labels.SelectorFromSet(labelset)) + aggregatedRequests, err := g.reportChangeRequestLister.ReportChangeRequests(config.KyvernoNamespace).List(labels.SelectorFromSet(labelset)) if err != nil { return err } @@ -399,7 +399,7 @@ func (g *ReportGenerator) aggregateReports(namespace string) ( } selector := labels.SelectorFromSet(labels.Set(map[string]string{resourceLabelNamespace: namespace})) - requests, err := g.reportChangeRequestLister.ReportChangeRequests(config.KubePolicyNamespace).List(selector) + requests, err := g.reportChangeRequestLister.ReportChangeRequests(config.KyvernoNamespace).List(selector) if err != nil { return nil, nil, fmt.Errorf("unable to list reportChangeRequests within namespace %s: %v", ns, err) } @@ -554,7 +554,7 @@ func (g *ReportGenerator) cleanupReportRequets(requestsGeneral interface{}) { defer g.log.V(5).Info("successfully cleaned up report requests") if requests, ok := requestsGeneral.([]*changerequest.ReportChangeRequest); ok { for _, request := range requests { - if err := g.dclient.DeleteResource(request.APIVersion, "ReportChangeRequest", config.KubePolicyNamespace, request.Name, false); err != nil { + if err := g.dclient.DeleteResource(request.APIVersion, "ReportChangeRequest", config.KyvernoNamespace, request.Name, false); err != nil { if !apierrors.IsNotFound(err) { g.log.Error(err, "failed to delete report request") } diff --git a/pkg/policyreport/reportrequest.go b/pkg/policyreport/reportrequest.go index c1ba319b01..46adacab07 100755 --- a/pkg/policyreport/reportrequest.go +++ b/pkg/policyreport/reportrequest.go @@ -293,10 +293,10 @@ func (gen *Generator) sync(reportReq *unstructured.Unstructured, info Info) erro return updateReportChangeRequest(gen.dclient, old, reportReq, logger) } - old, err := gen.reportChangeRequestLister.ReportChangeRequests(config.KubePolicyNamespace).Get(reportReq.GetName()) + old, err := gen.reportChangeRequestLister.ReportChangeRequests(config.KyvernoNamespace).Get(reportReq.GetName()) if err != nil { if apierrors.IsNotFound(err) { - if _, err = gen.dclient.CreateResource(reportReq.GetAPIVersion(), reportReq.GetKind(), config.KubePolicyNamespace, reportReq, false); err != nil { + if _, err = gen.dclient.CreateResource(reportReq.GetAPIVersion(), reportReq.GetKind(), config.KyvernoNamespace, reportReq, false); err != nil { if !apierrors.IsNotFound(err) { return fmt.Errorf("failed to create ReportChangeRequest: %v", err) } @@ -333,7 +333,7 @@ func updateReportChangeRequest(dClient *client.Client, old interface{}, new *uns return nil } - if _, err = dClient.UpdateResource(new.GetAPIVersion(), new.GetKind(), config.KubePolicyNamespace, new, false); err != nil { + if _, err = dClient.UpdateResource(new.GetAPIVersion(), new.GetKind(), config.KyvernoNamespace, new, false); err != nil { return fmt.Errorf("failed to update report request: %v", err) } diff --git a/pkg/webhookconfig/checker.go b/pkg/webhookconfig/checker.go deleted file mode 100644 index 9e0ce97bdd..0000000000 --- a/pkg/webhookconfig/checker.go +++ /dev/null @@ -1,85 +0,0 @@ -package webhookconfig - -import ( - "fmt" - "sync" - - "github.com/kyverno/kyverno/pkg/config" - admregapi "k8s.io/api/admissionregistration/v1beta1" - errorsapi "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func (wrc *WebhookRegistrationClient) constructVerifyMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { - return &admregapi.MutatingWebhookConfiguration{ - ObjectMeta: v1.ObjectMeta{ - Name: config.VerifyMutatingWebhookConfigurationName, - OwnerReferences: []v1.OwnerReference{ - wrc.constructOwner(), - }, - }, - Webhooks: []admregapi.MutatingWebhook{ - generateMutatingWebhook( - config.VerifyMutatingWebhookName, - config.VerifyMutatingWebhookServicePath, - caData, - true, - wrc.timeoutSeconds, - []string{"deployments/*"}, - "apps", - "v1", - []admregapi.OperationType{admregapi.Update}, - ), - }, - } -} - -func (wrc *WebhookRegistrationClient) constructDebugVerifyMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { - logger := wrc.log - url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.VerifyMutatingWebhookServicePath) - logger.V(4).Info("Debug VerifyMutatingWebhookConfig is registered with url", "url", url) - return &admregapi.MutatingWebhookConfiguration{ - ObjectMeta: v1.ObjectMeta{ - Name: config.VerifyMutatingWebhookConfigurationDebugName, - }, - Webhooks: []admregapi.MutatingWebhook{ - generateDebugMutatingWebhook( - config.VerifyMutatingWebhookName, - url, - caData, - true, - wrc.timeoutSeconds, - []string{"deployments/*"}, - "apps", - "v1", - []admregapi.OperationType{admregapi.Update}, - ), - }, - } -} - -func (wrc *WebhookRegistrationClient) removeVerifyWebhookMutatingWebhookConfig(wg *sync.WaitGroup) { - defer wg.Done() - - var err error - var mutatingConfig string - if wrc.serverIP != "" { - mutatingConfig = config.VerifyMutatingWebhookConfigurationDebugName - } else { - mutatingConfig = config.VerifyMutatingWebhookConfigurationName - } - - logger := wrc.log.WithValues("name", mutatingConfig) - err = wrc.client.DeleteResource("", MutatingWebhookConfigurationKind, "", mutatingConfig, false) - if errorsapi.IsNotFound(err) { - logger.V(5).Info("verify webhook configuration not found") - return - } - - if err != nil { - logger.Error(err, "failed to delete verify webhook configuration") - return - } - - logger.V(4).Info("successfully deleted verify webhook configuration") -} diff --git a/pkg/webhookconfig/common.go b/pkg/webhookconfig/common.go index 1035956ccd..bfe208797f 100644 --- a/pkg/webhookconfig/common.go +++ b/pkg/webhookconfig/common.go @@ -9,7 +9,7 @@ import ( rest "k8s.io/client-go/rest" ) -func (wrc *WebhookRegistrationClient) readCaData() []byte { +func (wrc *Register) readCaData() []byte { logger := wrc.log var caData []byte // Check if ca is defined in the secret tls-ca @@ -45,7 +45,7 @@ func extractCA(config *rest.Config) (result []byte) { return config.TLSClientConfig.CAData } -func (wrc *WebhookRegistrationClient) constructOwner() v1.OwnerReference { +func (wrc *Register) constructOwner() v1.OwnerReference { logger := wrc.log kubePolicyDeployment, err := wrc.client.GetKubePolicyDeployment() @@ -126,41 +126,6 @@ func generateDebugValidatingWebhook(name, url string, caData []byte, validate bo } } -// func generateWebhook(name, servicePath string, caData []byte, validation bool, timeoutSeconds int32, resource, apiGroups, apiVersions string, operationTypes []admregapi.OperationType) admregapi.Webhook { -// sideEffect := admregapi.SideEffectClassNoneOnDryRun -// failurePolicy := admregapi.Ignore -// return admregapi.Webhook{ -// Name: name, -// ClientConfig: admregapi.WebhookClientConfig{ -// Service: &admregapi.ServiceReference{ -// Namespace: config.KubePolicyNamespace, -// Name: config.WebhookServiceName, -// Path: &servicePath, -// }, -// CABundle: caData, -// }, -// SideEffects: &sideEffect, -// Rules: []admregapi.RuleWithOperations{ -// admregapi.RuleWithOperations{ -// Operations: operationTypes, -// Rule: admregapi.Rule{ -// APIGroups: []string{ -// apiGroups, -// }, -// APIVersions: []string{ -// apiVersions, -// }, -// Resources: []string{ -// resource, -// }, -// }, -// }, -// }, -// AdmissionReviewVersions: []string{"v1beta1"}, -// TimeoutSeconds: &timeoutSeconds, -// FailurePolicy: &failurePolicy, -// } -// } // mutating webhook func generateMutatingWebhook(name, servicePath string, caData []byte, validation bool, timeoutSeconds int32, resources []string, apiGroups, apiVersions string, operationTypes []admregapi.OperationType) admregapi.MutatingWebhook { @@ -173,8 +138,8 @@ func generateMutatingWebhook(name, servicePath string, caData []byte, validation Name: name, ClientConfig: admregapi.WebhookClientConfig{ Service: &admregapi.ServiceReference{ - Namespace: config.KubePolicyNamespace, - Name: config.WebhookServiceName, + Namespace: config.KyvernoNamespace, + Name: config.KyvernoServiceName, Path: &servicePath, }, CABundle: caData, @@ -208,8 +173,8 @@ func generateValidatingWebhook(name, servicePath string, caData []byte, validati Name: name, ClientConfig: admregapi.WebhookClientConfig{ Service: &admregapi.ServiceReference{ - Namespace: config.KubePolicyNamespace, - Name: config.WebhookServiceName, + Namespace: config.KyvernoNamespace, + Name: config.KyvernoServiceName, Path: &servicePath, }, CABundle: caData, diff --git a/pkg/webhookconfig/monitor.go b/pkg/webhookconfig/monitor.go new file mode 100644 index 0000000000..9c6696de2e --- /dev/null +++ b/pkg/webhookconfig/monitor.go @@ -0,0 +1,123 @@ +package webhookconfig + +import ( + "fmt" + "sync" + "time" + + "github.com/go-logr/logr" + dclient "github.com/kyverno/kyverno/pkg/dclient" + "github.com/kyverno/kyverno/pkg/event" +) + +//maxRetryCount defines the max deadline count +const ( + tickerInterval time.Duration = 10 * time.Second + idleCheckInterval time.Duration = 60 * time.Second + idleDeadline time.Duration = idleCheckInterval * 2 +) + +// Monitor stores the last webhook request time and monitors registered webhooks. +// +// If a webhook is not received in the idleCheckInterval the monitor triggers a +// change in the Kyverno deployment to force a webhook request. If no requests +// are received after idleDeadline the webhooks are deleted and re-registered. +// +// Webhook configurations are checked every tickerInterval. Currently the check +// only queries for the expected resource name, and does not compare other details +// like the webhook settings. +// +type Monitor struct { + t time.Time + mu sync.RWMutex + log logr.Logger +} + +//NewMonitor returns a new instance of LastRequestTime store +func NewMonitor(log logr.Logger) *Monitor { + return &Monitor{ + t: time.Now(), + log: log, + } +} + +//Time returns the last request time +func (t *Monitor) Time() time.Time { + t.mu.RLock() + defer t.mu.RUnlock() + return t.t +} + +//SetTime updates the last request time +func (t *Monitor) SetTime(tm time.Time) { + t.mu.Lock() + defer t.mu.Unlock() + + t.t = tm +} + +//Run runs the checker and verify the resource update +func (t *Monitor) Run(register *Register, eventGen event.Interface, client *dclient.Client, stopCh <-chan struct{}) { + logger := t.log + logger.V(4).Info("starting webhook monitor", "interval", idleCheckInterval) + status := newStatusControl(client, eventGen, logger.WithName("WebhookStatusControl")) + + ticker := time.NewTicker(tickerInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + + if err := register.Check(); err != nil { + t.log.Error(err,"missing webhooks") + if err := register.Register(); err != nil { + logger.Error(err,"failed to register webhooks") + } + + continue + } + + timeDiff := time.Since(t.Time()) + if timeDiff > idleDeadline { + err := fmt.Errorf("admission control configuration error") + logger.Error(err, "webhook check failed", "deadline", idleDeadline) + if err := status.failure(); err != nil { + logger.Error(err, "failed to annotate deployment webhook status to failure") + } + + cleanUp := make(chan struct{}) + register.Remove(cleanUp) + <-cleanUp + + if err:= register.Register(); err != nil { + logger.Error(err, "Failed to register webhooks") + } + + continue + } + + if timeDiff > idleCheckInterval { + logger.V(1).Info("webhook idle time exceeded", "deadline", idleCheckInterval) + + // send request to update the Kyverno deployment + if err := status.IncrementAnnotation(); err != nil { + logger.Error(err, "failed to annotate deployment for webhook status") + } + + continue + } + + // if the status was false before then we update it to true + // send request to update the Kyverno deployment + if err := status.success(); err != nil { + logger.Error(err, "failed to annotate deployment webhook status to success") + } + + case <-stopCh: + // handler termination signal + logger.V(2).Info("stopping webhook monitor") + return + } + } +} diff --git a/pkg/webhookconfig/policy.go b/pkg/webhookconfig/policy.go index 7ccc51cc65..0010270f1e 100644 --- a/pkg/webhookconfig/policy.go +++ b/pkg/webhookconfig/policy.go @@ -8,7 +8,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func (wrc *WebhookRegistrationClient) contructPolicyValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration { +func (wrc *Register) contructPolicyValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration { return &admregapi.ValidatingWebhookConfiguration{ ObjectMeta: v1.ObjectMeta{ @@ -33,7 +33,7 @@ func (wrc *WebhookRegistrationClient) contructPolicyValidatingWebhookConfig(caDa } } -func (wrc *WebhookRegistrationClient) contructDebugPolicyValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration { +func (wrc *Register) contructDebugPolicyValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration { logger := wrc.log url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.PolicyValidatingWebhookServicePath) logger.V(4).Info("Debug PolicyValidatingWebhookConfig is registered with url ", "url", url) @@ -58,7 +58,7 @@ func (wrc *WebhookRegistrationClient) contructDebugPolicyValidatingWebhookConfig } } -func (wrc *WebhookRegistrationClient) contructPolicyMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { +func (wrc *Register) contructPolicyMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { return &admregapi.MutatingWebhookConfiguration{ ObjectMeta: v1.ObjectMeta{ Name: config.PolicyMutatingWebhookConfigurationName, @@ -81,7 +81,8 @@ func (wrc *WebhookRegistrationClient) contructPolicyMutatingWebhookConfig(caData }, } } -func (wrc *WebhookRegistrationClient) contructDebugPolicyMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { + +func (wrc *Register) contructDebugPolicyMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { logger := wrc.log url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.PolicyMutatingWebhookServicePath) logger.V(4).Info("Debug PolicyMutatingWebhookConfig is registered with url ", "url", url) diff --git a/pkg/webhookconfig/registration.go b/pkg/webhookconfig/registration.go index 6510af37d9..42c6d48a85 100644 --- a/pkg/webhookconfig/registration.go +++ b/pkg/webhookconfig/registration.go @@ -3,6 +3,8 @@ package webhookconfig import ( "errors" "fmt" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strings" "sync" "time" @@ -15,318 +17,310 @@ import ( ) const ( - //MutatingWebhookConfigurationKind defines the kind for MutatingWebhookConfiguration - MutatingWebhookConfigurationKind string = "MutatingWebhookConfiguration" - //ValidatingWebhookConfigurationKind defines the kind for ValidatingWebhookConfiguration - ValidatingWebhookConfigurationKind string = "ValidatingWebhookConfiguration" + kindMutating string = "MutatingWebhookConfiguration" + kindValidating string = "ValidatingWebhookConfiguration" ) -// WebhookRegistrationClient is client for registration webhooks on cluster -type WebhookRegistrationClient struct { +// Register manages webhook registration. There are five webhooks: +// 1. Policy Validation +// 2. Policy Mutation +// 3. Resource Validation +// 4. Resource Mutation +// 5. Webhook Status Mutation +type Register struct { client *client.Client clientConfig *rest.Config - // serverIP should be used if running Kyverno out of clutser - serverIP string + serverIP string // when running outside a cluster timeoutSeconds int32 log logr.Logger } -// NewWebhookRegistrationClient creates new WebhookRegistrationClient instance -func NewWebhookRegistrationClient( +// NewRegister creates new Register instance +func NewRegister( clientConfig *rest.Config, client *client.Client, serverIP string, webhookTimeout int32, - log logr.Logger) *WebhookRegistrationClient { - return &WebhookRegistrationClient{ + log logr.Logger) *Register { + return &Register{ clientConfig: clientConfig, client: client, serverIP: serverIP, timeoutSeconds: webhookTimeout, - log: log.WithName("WebhookRegistrationClient"), + log: log.WithName("Register"), } } // Register creates admission webhooks configs on cluster -func (wrc *WebhookRegistrationClient) Register() error { - logger := wrc.log.WithName("Register") +func (wrc *Register) Register() error { + logger := wrc.log if wrc.serverIP != "" { - logger.V(4).Info("Registering webhook", "url", fmt.Sprintf("https://%s", wrc.serverIP)) + logger.Info("Registering webhook", "url", fmt.Sprintf("https://%s", wrc.serverIP)) } - // For the case if cluster already has this configs - // remove previously create webhookconfigurations if any - // webhook configurations are created dynamically based on the policy resources - wrc.removeWebhookConfigurations() - - // create Verify mutating webhook configuration resource - // that is used to check if admission control is enabled or not + errors := make([]string, 0) if err := wrc.createVerifyMutatingWebhookConfiguration(); err != nil { + errors = append(errors, err.Error()) + } + + if err := wrc.createPolicyValidatingWebhookConfiguration(); err != nil { + errors = append(errors, err.Error()) + } + + if err := wrc.createPolicyMutatingWebhookConfiguration(); err != nil { + errors = append(errors, err.Error()) + } + + if err := wrc.createResourceValidatingWebhookConfiguration(); err != nil { + errors = append(errors, err.Error()) + } + + if err := wrc.createResourceMutatingWebhookConfiguration(); err != nil { + errors = append(errors, err.Error()) + } + + if len(errors) > 0 { + return fmt.Errorf("%s", strings.Join(errors, ",")) + } + + return nil +} + +// CheckWebhooks returns an error if any of the webhooks are not configured +func (wrc *Register) Check() error { + + if _, err := wrc.client.GetResource("", kindMutating, "", wrc.getVerifyWebhookMutatingWebhookName()); err != nil { return err } - // Static Webhook configuration on Policy CRD - // create Policy CRD validating webhook configuration resource - // used for validating Policy CR - if err := wrc.createPolicyValidatingWebhookConfiguration(); err != nil { + if _, err := wrc.client.GetResource("", kindMutating, "", wrc.getResourceMutatingWebhookConfigName()); err != nil { return err } - // create Policy CRD validating webhook configuration resource - // used for defauling values in Policy CR - if err := wrc.createPolicyMutatingWebhookConfiguration(); err != nil { + + if _, err := wrc.client.GetResource("", kindValidating, "", wrc.getResourceValidatingWebhookConfigName()); err != nil { + return err + } + + if _, err := wrc.client.GetResource("", kindMutating, "", wrc.getPolicyMutatingWebhookConfigurationName()); err != nil { + return err + } + + if _, err := wrc.client.GetResource("", kindValidating, "", wrc.getPolicyValidatingWebhookConfigurationName()); err != nil { return err } return nil } -// RemoveWebhookConfigurations removes webhook configurations for reosurces and policy -// called during webhook server shutdown -func (wrc *WebhookRegistrationClient) RemoveWebhookConfigurations(cleanUp chan<- struct{}) { - //TODO: dupliate, but a placeholder to perform more error handlind during cleanup +// Remove removes all webhook configurations +func (wrc *Register) Remove(cleanUp chan<- struct{}) { wrc.removeWebhookConfigurations() - // close channel to notify cleanup is complete close(cleanUp) } -//CreateResourceMutatingWebhookConfiguration create a Mutatingwebhookconfiguration resource for all resource type -// used to forward request to kyverno webhooks to apply policeis -// Mutationg webhook is be used for Mutating purpose -func (wrc *WebhookRegistrationClient) CreateResourceMutatingWebhookConfiguration() error { - logger := wrc.log.WithValues("kind", MutatingWebhookConfigurationKind) +func (wrc *Register) createResourceMutatingWebhookConfiguration() error { var caData []byte var config *admregapi.MutatingWebhookConfiguration - // read CA data from - // 1) secret(config) - // 2) kubeconfig if caData = wrc.readCaData(); caData == nil { return errors.New("Unable to extract CA data from configuration") } - // if serverIP is specified we assume its debug mode + if wrc.serverIP != "" { - // debug mode - // clientConfig - URL config = wrc.constructDebugMutatingWebhookConfig(caData) } else { - // clientConfig - service config = wrc.constructMutatingWebhookConfig(caData) } - _, err := wrc.client.CreateResource("", MutatingWebhookConfigurationKind, "", *config, false) + + logger := wrc.log.WithValues("kind", kindMutating, "name", config.Name) + + _, err := wrc.client.CreateResource("", kindMutating, "", *config, false) if errorsapi.IsAlreadyExists(err) { logger.V(6).Info("resource mutating webhook configuration already exists", "name", config.Name) return nil } + if err != nil { logger.Error(err, "failed to create resource mutating webhook configuration", "name", config.Name) return err } - logger.V(2).Info("created mutating webhook", "name", config.Name) + logger.Info("created webhook") return nil } -//CreateResourceValidatingWebhookConfiguration ... -func (wrc *WebhookRegistrationClient) CreateResourceValidatingWebhookConfiguration() error { +func (wrc *Register) createResourceValidatingWebhookConfiguration() error { var caData []byte var config *admregapi.ValidatingWebhookConfiguration if caData = wrc.readCaData(); caData == nil { return errors.New("Unable to extract CA data from configuration") } - // if serverIP is specified we assume its debug mode if wrc.serverIP != "" { - // debug mode - // clientConfig - URL config = wrc.constructDebugValidatingWebhookConfig(caData) } else { - // clientConfig - service config = wrc.constructValidatingWebhookConfig(caData) } - logger := wrc.log.WithValues("kind", ValidatingWebhookConfigurationKind, "name", config.Name) - _, err := wrc.client.CreateResource("", ValidatingWebhookConfigurationKind, "", *config, false) + logger := wrc.log.WithValues("kind", kindValidating, "name", config.Name) + + _, err := wrc.client.CreateResource("", kindValidating, "", *config, false) if errorsapi.IsAlreadyExists(err) { logger.V(6).Info("resource validating webhook configuration already exists", "name", config.Name) return nil } + if err != nil { logger.Error(err, "failed to create resource") return err } - logger.V(2).Info("created validating webhook", "name", config.Name) + logger.Info("created webhook") return nil } //registerPolicyValidatingWebhookConfiguration create a Validating webhook configuration for Policy CRD -func (wrc *WebhookRegistrationClient) createPolicyValidatingWebhookConfiguration() error { +func (wrc *Register) createPolicyValidatingWebhookConfiguration() error { var caData []byte var config *admregapi.ValidatingWebhookConfiguration - // read CA data from - // 1) secret(config) - // 2) kubeconfig + // read certificate data if caData = wrc.readCaData(); caData == nil { return errors.New("Unable to extract CA data from configuration") } - // if serverIP is specified we assume its debug mode if wrc.serverIP != "" { - // debug mode - // clientConfig - URL config = wrc.contructDebugPolicyValidatingWebhookConfig(caData) } else { - // clientConfig - service config = wrc.contructPolicyValidatingWebhookConfig(caData) } - logger := wrc.log.WithValues("kind", ValidatingWebhookConfigurationKind, "name", config.Name) - // create validating webhook configuration resource - if _, err := wrc.client.CreateResource("", ValidatingWebhookConfigurationKind, "", *config, false); err != nil { + if _, err := wrc.client.CreateResource("", kindValidating, "", *config, false); err != nil { + if errorsapi.IsAlreadyExists(err) { + wrc.log.V(6).Info("webhook already exists", "kind", kindValidating, "name", config.Name) + return nil + } + return err } - logger.V(4).Info("created resource") + + wrc.log.Info("created webhook", "kind", kindValidating, "name", config.Name) return nil } -func (wrc *WebhookRegistrationClient) createPolicyMutatingWebhookConfiguration() error { +func (wrc *Register) createPolicyMutatingWebhookConfiguration() error { var caData []byte var config *admregapi.MutatingWebhookConfiguration - // read CA data from - // 1) secret(config) - // 2) kubeconfig + if caData = wrc.readCaData(); caData == nil { return errors.New("Unable to extract CA data from configuration") } - // if serverIP is specified we assume its debug mode if wrc.serverIP != "" { - // debug mode - // clientConfig - URL config = wrc.contructDebugPolicyMutatingWebhookConfig(caData) } else { - // clientConfig - service config = wrc.contructPolicyMutatingWebhookConfig(caData) } // create mutating webhook configuration resource - if _, err := wrc.client.CreateResource("", MutatingWebhookConfigurationKind, "", *config, false); err != nil { + if _, err := wrc.client.CreateResource("", kindMutating, "", *config, false); err != nil { + if errorsapi.IsAlreadyExists(err) { + wrc.log.V(6).Info("webhook already exists", "kind", kindMutating, "name", config.Name) + return nil + } + return err } - wrc.log.V(4).Info("reated Mutating Webhook Configuration", "name", config.Name) + + wrc.log.Info("created webhook", "kind", kindMutating, "name", config.Name) return nil } -func (wrc *WebhookRegistrationClient) createVerifyMutatingWebhookConfiguration() error { +func (wrc *Register) createVerifyMutatingWebhookConfiguration() error { var caData []byte var config *admregapi.MutatingWebhookConfiguration - // read CA data from - // 1) secret(config) - // 2) kubeconfig if caData = wrc.readCaData(); caData == nil { return errors.New("Unable to extract CA data from configuration") } - // if serverIP is specified we assume its debug mode if wrc.serverIP != "" { - // debug mode - // clientConfig - URL config = wrc.constructDebugVerifyMutatingWebhookConfig(caData) } else { - // clientConfig - service config = wrc.constructVerifyMutatingWebhookConfig(caData) } - // create mutating webhook configuration resource - if _, err := wrc.client.CreateResource("", MutatingWebhookConfigurationKind, "", *config, false); err != nil { + if _, err := wrc.client.CreateResource("", kindMutating, "", *config, false); err != nil { + if errorsapi.IsAlreadyExists(err) { + wrc.log.V(6).Info("webhook already exists", "kind", kindMutating, "name", config.Name) + return nil + } + return err } - wrc.log.V(4).Info("reated Mutating Webhook Configuration", "name", config.Name) + wrc.log.Info("created webhook", "kind", kindMutating, "name", config.Name) return nil } -// DeregisterAll deletes webhook configs from cluster -// This function does not fail on error: -// Register will fail if the config exists, so there is no need to fail on error -func (wrc *WebhookRegistrationClient) removeWebhookConfigurations() { +func (wrc *Register) removeWebhookConfigurations() { startTime := time.Now() - wrc.log.Info("removing prior webhook configurations") + wrc.log.Info("deleting all webhook configurations") defer func() { - wrc.log.V(4).Info("removed webhookcongfigurations", "processingTime", time.Since(startTime).String()) + wrc.log.V(4).Info("removed webhook configurations", "processingTime", time.Since(startTime).String()) }() var wg sync.WaitGroup - wg.Add(5) - // mutating and validating webhook configuration for Kubernetes resources go wrc.removeResourceMutatingWebhookConfiguration(&wg) go wrc.removeResourceValidatingWebhookConfiguration(&wg) - - // mutating and validating webhook configurtion for Policy CRD resource go wrc.removePolicyMutatingWebhookConfiguration(&wg) go wrc.removePolicyValidatingWebhookConfiguration(&wg) - - // mutating webhook configuration for verifying webhook go wrc.removeVerifyWebhookMutatingWebhookConfig(&wg) - // wait for the removal go routines to return wg.Wait() } -// wrapper to handle wait group -// TODO: re-work with RemoveResourceMutatingWebhookConfiguration, as the only difference is wg handling -func (wrc *WebhookRegistrationClient) removeResourceMutatingWebhookConfiguration(wg *sync.WaitGroup) { - defer wg.Done() - wrc.RemoveResourceMutatingWebhookConfiguration() -} -func (wrc *WebhookRegistrationClient) removeResourceValidatingWebhookConfiguration(wg *sync.WaitGroup) { - defer wg.Done() - wrc.RemoveResourceValidatingWebhookConfiguration() -} - -func (wrc *WebhookRegistrationClient) removePolicyMutatingWebhookConfiguration(wg *sync.WaitGroup) { +func (wrc *Register) removePolicyMutatingWebhookConfiguration(wg *sync.WaitGroup) { defer wg.Done() - var mutatingConfig string - if wrc.serverIP != "" { - mutatingConfig = config.PolicyMutatingWebhookConfigurationDebugName - } else { - mutatingConfig = config.PolicyMutatingWebhookConfigurationName - } + mutatingConfig := wrc.getPolicyMutatingWebhookConfigurationName() - logger := wrc.log.WithValues("name", mutatingConfig) - err := wrc.client.DeleteResource("", MutatingWebhookConfigurationKind, "", mutatingConfig, false) + logger := wrc.log.WithValues("kind", kindMutating, "name", mutatingConfig) + err := wrc.client.DeleteResource("", kindMutating, "", mutatingConfig, false) if errorsapi.IsNotFound(err) { logger.V(5).Info("policy mutating webhook configuration not found") return - } + } if err != nil { logger.Error(err, "failed to delete policy mutating webhook configuration") return } - logger.V(4).Info("successfully deleted policy mutating webhook configutation") + logger.Info("webhook configuration deleted") } -func (wrc *WebhookRegistrationClient) removePolicyValidatingWebhookConfiguration(wg *sync.WaitGroup) { +func (wrc *Register) getPolicyMutatingWebhookConfigurationName() string { + var mutatingConfig string + if wrc.serverIP != "" { + mutatingConfig = config.PolicyMutatingWebhookConfigurationDebugName + } else { + mutatingConfig = config.PolicyMutatingWebhookConfigurationName + } + return mutatingConfig +} + +func (wrc *Register) removePolicyValidatingWebhookConfiguration(wg *sync.WaitGroup) { defer wg.Done() - var validatingConfig string - if wrc.serverIP != "" { - validatingConfig = config.PolicyValidatingWebhookConfigurationDebugName - } else { - validatingConfig = config.PolicyValidatingWebhookConfigurationName - } + validatingConfig := wrc.getPolicyValidatingWebhookConfigurationName() - logger := wrc.log.WithValues("name", validatingConfig) + logger := wrc.log.WithValues("kind", kindValidating, "name", validatingConfig) logger.V(4).Info("removing validating webhook configuration") - err := wrc.client.DeleteResource("", ValidatingWebhookConfigurationKind, "", validatingConfig, false) + err := wrc.client.DeleteResource("", kindValidating, "", validatingConfig, false) if errorsapi.IsNotFound(err) { logger.V(5).Info("policy validating webhook configuration not found") return @@ -337,10 +331,101 @@ func (wrc *WebhookRegistrationClient) removePolicyValidatingWebhookConfiguration return } - logger.V(4).Info("successfully deleted policy validating webhook configutation") + logger.Info("webhook configuration deleted") } +func (wrc *Register) getPolicyValidatingWebhookConfigurationName() string { + var validatingConfig string + if wrc.serverIP != "" { + validatingConfig = config.PolicyValidatingWebhookConfigurationDebugName + } else { + validatingConfig = config.PolicyValidatingWebhookConfigurationName + } + return validatingConfig +} + +func (wrc *Register) constructVerifyMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { + return &admregapi.MutatingWebhookConfiguration{ + ObjectMeta: v1.ObjectMeta{ + Name: config.VerifyMutatingWebhookConfigurationName, + OwnerReferences: []v1.OwnerReference{ + wrc.constructOwner(), + }, + }, + Webhooks: []admregapi.MutatingWebhook{ + generateMutatingWebhook( + config.VerifyMutatingWebhookName, + config.VerifyMutatingWebhookServicePath, + caData, + true, + wrc.timeoutSeconds, + []string{"deployments/*"}, + "apps", + "v1", + []admregapi.OperationType{admregapi.Update}, + ), + }, + } +} + +func (wrc *Register) constructDebugVerifyMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { + logger := wrc.log + url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.VerifyMutatingWebhookServicePath) + logger.V(4).Info("Debug VerifyMutatingWebhookConfig is registered with url", "url", url) + return &admregapi.MutatingWebhookConfiguration{ + ObjectMeta: v1.ObjectMeta{ + Name: config.VerifyMutatingWebhookConfigurationDebugName, + }, + Webhooks: []admregapi.MutatingWebhook{ + generateDebugMutatingWebhook( + config.VerifyMutatingWebhookName, + url, + caData, + true, + wrc.timeoutSeconds, + []string{"deployments/*"}, + "apps", + "v1", + []admregapi.OperationType{admregapi.Update}, + ), + }, + } +} + +func (wrc *Register) removeVerifyWebhookMutatingWebhookConfig(wg *sync.WaitGroup) { + defer wg.Done() + + var err error + mutatingConfig := wrc.getVerifyWebhookMutatingWebhookName() + + logger := wrc.log.WithValues("kind", kindMutating, "name", mutatingConfig) + err = wrc.client.DeleteResource("", kindMutating, "", mutatingConfig, false) + if errorsapi.IsNotFound(err) { + logger.V(5).Info("verify webhook configuration not found") + return + } + + if err != nil { + logger.Error(err, "failed to delete verify webhook configuration") + return + } + + logger.Info("webhook configuration deleted") +} + +func (wrc *Register) getVerifyWebhookMutatingWebhookName() string { + var mutatingConfig string + if wrc.serverIP != "" { + mutatingConfig = config.VerifyMutatingWebhookConfigurationDebugName + } else { + mutatingConfig = config.VerifyMutatingWebhookConfigurationName + } + return mutatingConfig +} + + + // GetWebhookTimeOut returns the value of webhook timeout -func (wrc *WebhookRegistrationClient) GetWebhookTimeOut() time.Duration { +func (wrc *Register) GetWebhookTimeOut() time.Duration { return time.Duration(wrc.timeoutSeconds) } diff --git a/pkg/webhookconfig/registration_test.go b/pkg/webhookconfig/registration_test.go index 2188007f44..82afc457f7 100644 --- a/pkg/webhookconfig/registration_test.go +++ b/pkg/webhookconfig/registration_test.go @@ -2,27 +2,38 @@ package webhookconfig import ( "bytes" - "io/ioutil" "testing" "gotest.tools/assert" rest "k8s.io/client-go/rest" ) -func TestExtractCA_EmptyBundle(t *testing.T) { - CAFile := "resources/CAFile" +var cert = ` +-----BEGIN CERTIFICATE----- +V2VsY29tZSB0byBUaGUgUnVzdCBQcm9ncmFtbWluZyBMYW5ndWFnZSwgY +W4gaW50cm9kdWN0b3J5IGJvb2sgYWJvdXQgUnVzdC4gVGhlIFJ1c3QgcH +JvZ3JhbW1pbmcgbGFuZ3VhZ2UgaGVscHMgeW91IHdyaXRlIGZhc3Rlciw +gbW9yZSByZWxpYWJsZSBzb2Z0d2FyZS4gSGlnaC1sZXZlbCBlcmdvbm9t +aWNzIGFuZCBsb3ctbGV2ZWwgY29udHJvbCBhcmUgb2Z0ZW4gYXQgb2Rkc +yBpbiBwcm9ncmFtbWluZyBsYW5ndWFnZSBkZXNpZ247IFJ1c3QgY2hhbG +xlbmdlcyB0aGF0IGNvbmZsaWN0LiBUaHJvdWdoIGJhbGFuY2luZyBwb3d +lcmZ1bCB0ZWNobmljYWwgY2FwYWNpdHkgYW5kIGEgZ3JlYXQgZGV2ZWxv +cGVyIGV4cGVyaWVuY2UsIFJ1c3QgZ2l2ZXMgeW91IHRoZSBvcHRpb24gd +G8gY29udHJvbCBsb3ctbGV2ZWwgZGV0YWlscyAoc3VjaCBhcyBtZW1vcn +kgdXNhZ2UpIHdpdGhvdXQgYWxsIHRoZSBoYXNzbGUgdHJhZGl0aW9uYWx +seSBhc3NvY2lhdGVkIHdpdGggc3VjaCBjb250cm9sLgyzmqp31l8rqr1== +-----END CERTIFICATE----- +` +func TestExtractCA_EmptyBundle(t *testing.T) { config := &rest.Config{ TLSClientConfig: rest.TLSClientConfig{ - CAData: nil, - CAFile: CAFile, + CAData: []byte(cert), }, } - expected, err := ioutil.ReadFile(CAFile) - assert.Assert(t, err == nil) actual := extractCA(config) - assert.Assert(t, bytes.Equal(expected, actual)) + assert.Assert(t, bytes.Equal([]byte(cert), actual)) } func TestExtractCA_EmptyCAFile(t *testing.T) { diff --git a/pkg/webhookconfig/resource.go b/pkg/webhookconfig/resource.go index 389ada23e7..d7f7f8dfb2 100644 --- a/pkg/webhookconfig/resource.go +++ b/pkg/webhookconfig/resource.go @@ -2,6 +2,7 @@ package webhookconfig import ( "fmt" + "sync" "github.com/kyverno/kyverno/pkg/config" admregapi "k8s.io/api/admissionregistration/v1beta1" @@ -9,7 +10,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func (wrc *WebhookRegistrationClient) constructDebugMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { +func (wrc *Register) constructDebugMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { logger := wrc.log url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.MutatingWebhookServicePath) logger.V(4).Info("Debug MutatingWebhookConfig registered", "url", url) @@ -33,7 +34,7 @@ func (wrc *WebhookRegistrationClient) constructDebugMutatingWebhookConfig(caData } } -func (wrc *WebhookRegistrationClient) constructMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { +func (wrc *Register) constructMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { webhookCfg := generateMutatingWebhook( config.MutatingWebhookName, @@ -56,20 +57,21 @@ func (wrc *WebhookRegistrationClient) constructMutatingWebhookConfig(caData []by } } -//GetResourceMutatingWebhookConfigName returns the webhook configuration name -func (wrc *WebhookRegistrationClient) GetResourceMutatingWebhookConfigName() string { +//getResourceMutatingWebhookConfigName returns the webhook configuration name +func (wrc *Register) getResourceMutatingWebhookConfigName() string { if wrc.serverIP != "" { return config.MutatingWebhookConfigurationDebugName } return config.MutatingWebhookConfigurationName } -//RemoveResourceMutatingWebhookConfiguration removes mutating webhook configuration for all resources -func (wrc *WebhookRegistrationClient) RemoveResourceMutatingWebhookConfiguration() { - configName := wrc.GetResourceMutatingWebhookConfigName() - logger := wrc.log.WithValues("kind", MutatingWebhookConfigurationKind, "name", configName) +func (wrc *Register) removeResourceMutatingWebhookConfiguration(wg *sync.WaitGroup) { + defer wg.Done() + + configName := wrc.getResourceMutatingWebhookConfigName() + logger := wrc.log.WithValues("kind", kindMutating, "name", configName) // delete webhook configuration - err := wrc.client.DeleteResource("", MutatingWebhookConfigurationKind, "", configName, false) + err := wrc.client.DeleteResource("", kindMutating, "", configName, false) if errors.IsNotFound(err) { logger.V(4).Info("webhook configuration not found") return @@ -80,10 +82,10 @@ func (wrc *WebhookRegistrationClient) RemoveResourceMutatingWebhookConfiguration return } - logger.Info("mutating webhook configuration deleted") + logger.Info("webhook configuration deleted") } -func (wrc *WebhookRegistrationClient) constructDebugValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration { +func (wrc *Register) constructDebugValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration { url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.ValidatingWebhookServicePath) return &admregapi.ValidatingWebhookConfiguration{ @@ -106,7 +108,7 @@ func (wrc *WebhookRegistrationClient) constructDebugValidatingWebhookConfig(caDa } } -func (wrc *WebhookRegistrationClient) constructValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration { +func (wrc *Register) constructValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration { return &admregapi.ValidatingWebhookConfiguration{ ObjectMeta: v1.ObjectMeta{ Name: config.ValidatingWebhookConfigurationName, @@ -130,8 +132,8 @@ func (wrc *WebhookRegistrationClient) constructValidatingWebhookConfig(caData [] } } -// GetResourceValidatingWebhookConfigName returns the webhook configuration name -func (wrc *WebhookRegistrationClient) GetResourceValidatingWebhookConfigName() string { +// getResourceValidatingWebhookConfigName returns the webhook configuration name +func (wrc *Register) getResourceValidatingWebhookConfigName() string { if wrc.serverIP != "" { return config.ValidatingWebhookConfigurationDebugName } @@ -139,11 +141,12 @@ func (wrc *WebhookRegistrationClient) GetResourceValidatingWebhookConfigName() s return config.ValidatingWebhookConfigurationName } -// RemoveResourceValidatingWebhookConfiguration deletes an existing webhook configuration -func (wrc *WebhookRegistrationClient) RemoveResourceValidatingWebhookConfiguration() { - configName := wrc.GetResourceValidatingWebhookConfigName() - logger := wrc.log.WithValues("kind", ValidatingWebhookConfigurationKind, "name", configName) - err := wrc.client.DeleteResource("", ValidatingWebhookConfigurationKind, "", configName, false) +func (wrc *Register) removeResourceValidatingWebhookConfiguration(wg *sync.WaitGroup) { + defer wg.Done() + + configName := wrc.getResourceValidatingWebhookConfigName() + logger := wrc.log.WithValues("kind", kindValidating, "name", configName) + err := wrc.client.DeleteResource("", kindValidating, "", configName, false) if errors.IsNotFound(err) { logger.V(5).Info("webhook configuration not found") return @@ -154,6 +157,7 @@ func (wrc *WebhookRegistrationClient) RemoveResourceValidatingWebhookConfigurati return } - logger.Info("validating webhook configuration deleted") + logger.Info("webhook configuration deleted") return } + diff --git a/pkg/webhookconfig/resources/CAFile b/pkg/webhookconfig/resources/CAFile deleted file mode 100644 index 8e3c60a196..0000000000 --- a/pkg/webhookconfig/resources/CAFile +++ /dev/null @@ -1,14 +0,0 @@ ------BEGIN CERTIFICATE----- -V2VsY29tZSB0byBUaGUgUnVzdCBQcm9ncmFtbWluZyBMYW5ndWFnZSwgY -W4gaW50cm9kdWN0b3J5IGJvb2sgYWJvdXQgUnVzdC4gVGhlIFJ1c3QgcH -JvZ3JhbW1pbmcgbGFuZ3VhZ2UgaGVscHMgeW91IHdyaXRlIGZhc3Rlciw -gbW9yZSByZWxpYWJsZSBzb2Z0d2FyZS4gSGlnaC1sZXZlbCBlcmdvbm9t -aWNzIGFuZCBsb3ctbGV2ZWwgY29udHJvbCBhcmUgb2Z0ZW4gYXQgb2Rkc -yBpbiBwcm9ncmFtbWluZyBsYW5ndWFnZSBkZXNpZ247IFJ1c3QgY2hhbG -xlbmdlcyB0aGF0IGNvbmZsaWN0LiBUaHJvdWdoIGJhbGFuY2luZyBwb3d -lcmZ1bCB0ZWNobmljYWwgY2FwYWNpdHkgYW5kIGEgZ3JlYXQgZGV2ZWxv -cGVyIGV4cGVyaWVuY2UsIFJ1c3QgZ2l2ZXMgeW91IHRoZSBvcHRpb24gd -G8gY29udHJvbCBsb3ctbGV2ZWwgZGV0YWlscyAoc3VjaCBhcyBtZW1vcn -kgdXNhZ2UpIHdpdGhvdXQgYWxsIHRoZSBoYXNzbGUgdHJhZGl0aW9uYWx -seSBhc3NvY2lhdGVkIHdpdGggc3VjaCBjb250cm9sLgyzmqp31l8rqr1== ------END CERTIFICATE----- diff --git a/pkg/webhookconfig/rwebhookregister.go b/pkg/webhookconfig/rwebhookregister.go deleted file mode 100644 index 217792c1c2..0000000000 --- a/pkg/webhookconfig/rwebhookregister.go +++ /dev/null @@ -1,123 +0,0 @@ -package webhookconfig - -import ( - "time" - - "github.com/go-logr/logr" - checker "github.com/kyverno/kyverno/pkg/checker" - "github.com/tevino/abool" - mconfiginformer "k8s.io/client-go/informers/admissionregistration/v1beta1" - mconfiglister "k8s.io/client-go/listers/admissionregistration/v1beta1" - cache "k8s.io/client-go/tools/cache" -) - -//ResourceWebhookRegister manages the resource webhook registration -type ResourceWebhookRegister struct { - // pendingCreation indicates the status of resource webhook creation - pendingMutateWebhookCreation *abool.AtomicBool - pendingValidateWebhookCreation *abool.AtomicBool - LastReqTime *checker.LastReqTime - mwebhookconfigSynced cache.InformerSynced - vwebhookconfigSynced cache.InformerSynced - mWebhookConfigLister mconfiglister.MutatingWebhookConfigurationLister - vWebhookConfigLister mconfiglister.ValidatingWebhookConfigurationLister - webhookRegistrationClient *WebhookRegistrationClient - RunValidationInMutatingWebhook string - log logr.Logger -} - -// NewResourceWebhookRegister returns a new instance of ResourceWebhookRegister manager -func NewResourceWebhookRegister( - lastReqTime *checker.LastReqTime, - mconfigwebhookinformer mconfiginformer.MutatingWebhookConfigurationInformer, - vconfigwebhookinformer mconfiginformer.ValidatingWebhookConfigurationInformer, - webhookRegistrationClient *WebhookRegistrationClient, - runValidationInMutatingWebhook string, - log logr.Logger, -) *ResourceWebhookRegister { - return &ResourceWebhookRegister{ - pendingMutateWebhookCreation: abool.New(), - pendingValidateWebhookCreation: abool.New(), - LastReqTime: lastReqTime, - mwebhookconfigSynced: mconfigwebhookinformer.Informer().HasSynced, - mWebhookConfigLister: mconfigwebhookinformer.Lister(), - vwebhookconfigSynced: vconfigwebhookinformer.Informer().HasSynced, - vWebhookConfigLister: vconfigwebhookinformer.Lister(), - webhookRegistrationClient: webhookRegistrationClient, - RunValidationInMutatingWebhook: runValidationInMutatingWebhook, - log: log, - } -} - -//RegisterResourceWebhook registers a resource webhook -func (rww *ResourceWebhookRegister) RegisterResourceWebhook() { - timeDiff := time.Since(rww.LastReqTime.Time()) - if timeDiff < checker.DefaultDeadline { - if !rww.pendingMutateWebhookCreation.IsSet() { - go rww.createMutatingWebhook() - } - - if !rww.pendingValidateWebhookCreation.IsSet() { - go rww.createValidateWebhook() - } - } -} - -func (rww *ResourceWebhookRegister) createMutatingWebhook() { - rww.pendingMutateWebhookCreation.Set() - defer rww.pendingMutateWebhookCreation.UnSet() - - mutatingConfigName := rww.webhookRegistrationClient.GetResourceMutatingWebhookConfigName() - mutatingConfig, _ := rww.mWebhookConfigLister.Get(mutatingConfigName) - if mutatingConfig != nil { - rww.log.V(5).Info("mutating webhoook configuration exists", "name", mutatingConfigName) - } else { - err := rww.webhookRegistrationClient.CreateResourceMutatingWebhookConfiguration() - if err != nil { - rww.log.Error(err, "failed to create resource mutating webhook configuration, re-queue creation request") - rww.RegisterResourceWebhook() - return - } - } -} - -func (rww *ResourceWebhookRegister) createValidateWebhook() { - rww.pendingValidateWebhookCreation.Set() - defer rww.pendingValidateWebhookCreation.UnSet() - - if rww.RunValidationInMutatingWebhook == "true" { - rww.log.V(2).Info("validation is configured to run during mutate webhook") - return - } - - validatingConfigName := rww.webhookRegistrationClient.GetResourceValidatingWebhookConfigName() - validatingConfig, _ := rww.vWebhookConfigLister.Get(validatingConfigName) - if validatingConfig != nil { - rww.log.V(4).Info("validating webhoook configuration exists", "name", validatingConfigName) - } else { - err := rww.webhookRegistrationClient.CreateResourceValidatingWebhookConfiguration() - if err != nil { - rww.log.Error(err, "failed to create resource validating webhook configuration; re-queue creation request") - rww.RegisterResourceWebhook() - return - } - } -} - -//Run starts the ResourceWebhookRegister manager -func (rww *ResourceWebhookRegister) Run(stopCh <-chan struct{}) { - logger := rww.log - // wait for cache to populate first time - if !cache.WaitForCacheSync(stopCh, rww.mwebhookconfigSynced, rww.vwebhookconfigSynced) { - logger.Info("configuration: failed to sync webhook informer cache") - } -} - -// RemoveResourceWebhookConfiguration removes the resource webhook configurations -func (rww *ResourceWebhookRegister) RemoveResourceWebhookConfiguration() { - rww.webhookRegistrationClient.RemoveResourceMutatingWebhookConfiguration() - - if rww.RunValidationInMutatingWebhook != "true" { - rww.webhookRegistrationClient.RemoveResourceValidatingWebhookConfiguration() - } -} diff --git a/pkg/checker/status.go b/pkg/webhookconfig/status.go similarity index 76% rename from pkg/checker/status.go rename to pkg/webhookconfig/status.go index 6c12661727..d2a349d6e2 100644 --- a/pkg/checker/status.go +++ b/pkg/webhookconfig/status.go @@ -1,4 +1,4 @@ -package checker +package webhookconfig import ( "fmt" @@ -10,49 +10,39 @@ import ( "github.com/kyverno/kyverno/pkg/event" ) -var deployName string = config.KubePolicyDeploymentName -var deployNamespace string = config.KubePolicyNamespace +var deployName string = config.KyvernoDeploymentName +var deployNamespace string = config.KyvernoNamespace const annCounter string = "kyverno.io/generationCounter" const annWebhookStatus string = "kyverno.io/webhookActive" -//StatusInterface provides api to update webhook active annotations on kyverno deployments -type StatusInterface interface { - // Increments generation counter annotation - IncrementAnnotation() error - // update annotation to inform webhook is active - SuccessStatus() error - // update annotation to inform webhook is inactive - FailedStatus() error -} - -//StatusControl controls the webhook status -type StatusControl struct { +//statusControl controls the webhook status +type statusControl struct { client *dclient.Client eventGen event.Interface log logr.Logger } -//SuccessStatus ... -func (vc StatusControl) SuccessStatus() error { +//success ... +func (vc statusControl) success() error { return vc.setStatus("true") } -//FailedStatus ... -func (vc StatusControl) FailedStatus() error { +//failure ... +func (vc statusControl) failure() error { return vc.setStatus("false") } -// NewVerifyControl ... -func NewVerifyControl(client *dclient.Client, eventGen event.Interface, log logr.Logger) *StatusControl { - return &StatusControl{ +// NewStatusControl creates a new webhook status control +func newStatusControl(client *dclient.Client, eventGen event.Interface, log logr.Logger) *statusControl { + return &statusControl{ client: client, eventGen: eventGen, log: log, } } -func (vc StatusControl) setStatus(status string) error { +func (vc statusControl) setStatus(status string) error { logger := vc.log.WithValues("name", deployName, "namespace", deployNamespace) var ann map[string]string var err error @@ -105,7 +95,7 @@ func createStatusUpdateEvent(status string, eventGen event.Interface) { } //IncrementAnnotation ... -func (vc StatusControl) IncrementAnnotation() error { +func (vc statusControl) IncrementAnnotation() error { logger := vc.log var ann map[string]string var err error diff --git a/pkg/webhooks/generate/generate.go b/pkg/webhooks/generate/generate.go index 0e90f7f144..7130ea8cdd 100644 --- a/pkg/webhooks/generate/generate.go +++ b/pkg/webhooks/generate/generate.go @@ -115,14 +115,14 @@ func retryApplyResource(client *kyvernoclient.Clientset, Spec: grSpec, } - gr.SetNamespace(config.KubePolicyNamespace) + gr.SetNamespace(config.KyvernoNamespace) // Initial state "Pending" // TODO: status is not updated // gr.Status.State = kyverno.Pending // generate requests created in kyverno namespace isExist := false if action == v1beta1.Create || action == v1beta1.Update { - grList, err := client.KyvernoV1().GenerateRequests(config.KubePolicyNamespace).List(context.TODO(), metav1.ListOptions{}) + grList, err := client.KyvernoV1().GenerateRequests(config.KyvernoNamespace).List(context.TODO(), metav1.ListOptions{}) if err != nil { return err } @@ -135,7 +135,7 @@ func retryApplyResource(client *kyvernoclient.Clientset, v.Spec.Context = gr.Spec.Context v.Spec.Policy = gr.Spec.Policy v.Spec.Resource = gr.Spec.Resource - _, err = client.KyvernoV1().GenerateRequests(config.KubePolicyNamespace).Update(context.TODO(), &grList.Items[i], metav1.UpdateOptions{}) + _, err = client.KyvernoV1().GenerateRequests(config.KyvernoNamespace).Update(context.TODO(), &grList.Items[i], metav1.UpdateOptions{}) if err != nil { return err } @@ -144,7 +144,7 @@ func retryApplyResource(client *kyvernoclient.Clientset, } if !isExist { gr.SetGenerateName("gr-") - _, err = client.KyvernoV1().GenerateRequests(config.KubePolicyNamespace).Create(context.TODO(), &gr, metav1.CreateOptions{}) + _, err = client.KyvernoV1().GenerateRequests(config.KyvernoNamespace).Create(context.TODO(), &gr, metav1.CreateOptions{}) if err != nil { return err } diff --git a/pkg/webhooks/generation.go b/pkg/webhooks/generation.go index 250c35adf4..3ce4aa94ad 100644 --- a/pkg/webhooks/generation.go +++ b/pkg/webhooks/generation.go @@ -56,13 +56,13 @@ func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, polic engineResponse := engine.Generate(policyContext) for _, rule := range engineResponse.PolicyResponse.Rules { if !rule.Success { - grList, err := ws.kyvernoClient.KyvernoV1().GenerateRequests(config.KubePolicyNamespace).List(contextdefault.TODO(), metav1.ListOptions{}) + grList, err := ws.kyvernoClient.KyvernoV1().GenerateRequests(config.KyvernoNamespace).List(contextdefault.TODO(), metav1.ListOptions{}) if err != nil { logger.Error(err, "failed to list generate request") } for _, v := range grList.Items { if engineResponse.PolicyResponse.Policy == v.Spec.Policy && engineResponse.PolicyResponse.Resource.Name == v.Spec.Resource.Name && engineResponse.PolicyResponse.Resource.Kind == v.Spec.Resource.Kind && engineResponse.PolicyResponse.Resource.Namespace == v.Spec.Resource.Namespace { - err := ws.kyvernoClient.KyvernoV1().GenerateRequests(config.KubePolicyNamespace).Delete(contextdefault.TODO(), v.GetName(), metav1.DeleteOptions{}) + err := ws.kyvernoClient.KyvernoV1().GenerateRequests(config.KyvernoNamespace).Delete(contextdefault.TODO(), v.GetName(), metav1.DeleteOptions{}) if err != nil { logger.Error(err, "failed to update gr") } diff --git a/pkg/webhooks/policyvalidation.go b/pkg/webhooks/policyvalidation.go index a4ded2088b..878fd776c6 100644 --- a/pkg/webhooks/policyvalidation.go +++ b/pkg/webhooks/policyvalidation.go @@ -12,15 +12,14 @@ import ( //HandlePolicyValidation performs the validation check on policy resource func (ws *WebhookServer) policyValidation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { - logger := ws.log.WithValues("action", "policyvalidation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation) + logger := ws.log.WithValues("action", "policy validation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation) startTime := time.Now() logger.V(3).Info("start validating policy") defer logger.V(3).Info("finished validating policy", "time", time.Since(startTime).String()) - //TODO: can this happen? wont this be picked by OpenAPI spec schema ? if err := policyvalidate.Validate(request.Object.Raw, ws.client, false, ws.openAPIController); err != nil { - logger.Error(err, "failed to validate policy") + logger.Error(err, "policy validation errors") return &v1beta1.AdmissionResponse{ Allowed: false, Result: &metav1.Status{ @@ -29,9 +28,6 @@ func (ws *WebhookServer) policyValidation(request *v1beta1.AdmissionRequest) *v1 } } - // if the policy contains mutating & validation rules and it config does not exist we create one - // queue the request - ws.resourceWebhookWatcher.RegisterResourceWebhook() return &v1beta1.AdmissionResponse{ Allowed: true, } diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index b17e527a56..de0027fbdf 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -13,7 +13,6 @@ import ( "github.com/go-logr/logr" "github.com/julienschmidt/httprouter" v1 "github.com/kyverno/kyverno/pkg/api/kyverno/v1" - "github.com/kyverno/kyverno/pkg/checker" kyvernoclient "github.com/kyverno/kyverno/pkg/client/clientset/versioned" kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1" kyvernolister "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1" @@ -83,7 +82,7 @@ type WebhookServer struct { pCache policycache.Interface // webhook registration client - webhookRegistrationClient *webhookconfig.WebhookRegistrationClient + webhookRegister *webhookconfig.Register // API to send policy stats for aggregation statusListener policystatus.Listener @@ -95,7 +94,7 @@ type WebhookServer struct { cleanUp chan<- struct{} // last request time - lastReqTime *checker.LastReqTime + webhookMonitor *webhookconfig.Monitor // policy report generator prGenerator policyreport.GeneratorInterface @@ -103,11 +102,10 @@ type WebhookServer struct { // generate request generator grGenerator *generate.Generator - resourceWebhookWatcher *webhookconfig.ResourceWebhookRegister - auditHandler AuditHandler log logr.Logger + openAPIController *openapi.Controller supportMutateValidate bool @@ -129,12 +127,12 @@ func NewWebhookServer( crInformer rbacinformer.ClusterRoleInformer, eventGen event.Interface, pCache policycache.Interface, - webhookRegistrationClient *webhookconfig.WebhookRegistrationClient, + webhookRegistrationClient *webhookconfig.Register, + webhookMonitor *webhookconfig.Monitor, statusSync policystatus.Listener, configHandler config.Interface, prGenerator policyreport.GeneratorInterface, grGenerator *generate.Generator, - resourceWebhookWatcher *webhookconfig.ResourceWebhookRegister, auditHandler AuditHandler, supportMutateValidate bool, cleanUp chan<- struct{}, @@ -164,24 +162,23 @@ func NewWebhookServer( rLister: rInformer.Lister(), rSynced: rInformer.Informer().HasSynced, - crbLister: crbInformer.Lister(), - crLister: crInformer.Lister(), - crbSynced: crbInformer.Informer().HasSynced, - crSynced: crInformer.Informer().HasSynced, - eventGen: eventGen, - pCache: pCache, - webhookRegistrationClient: webhookRegistrationClient, - statusListener: statusSync, - configHandler: configHandler, - cleanUp: cleanUp, - lastReqTime: resourceWebhookWatcher.LastReqTime, - prGenerator: prGenerator, - grGenerator: grGenerator, - resourceWebhookWatcher: resourceWebhookWatcher, - auditHandler: auditHandler, - log: log, - openAPIController: openAPIController, - supportMutateValidate: supportMutateValidate, + crbLister: crbInformer.Lister(), + crLister: crInformer.Lister(), + crbSynced: crbInformer.Informer().HasSynced, + crSynced: crInformer.Informer().HasSynced, + eventGen: eventGen, + pCache: pCache, + webhookRegister: webhookRegistrationClient, + statusListener: statusSync, + configHandler: configHandler, + cleanUp: cleanUp, + webhookMonitor: webhookMonitor, + prGenerator: prGenerator, + grGenerator: grGenerator, + auditHandler: auditHandler, + log: log, + openAPIController: openAPIController, + supportMutateValidate: supportMutateValidate, resCache: resCache, } @@ -221,7 +218,7 @@ func NewWebhookServer( func (ws *WebhookServer) handlerFunc(handler func(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse, filter bool) http.HandlerFunc { return func(rw http.ResponseWriter, r *http.Request) { startTime := time.Now() - ws.lastReqTime.SetTime(startTime) + ws.webhookMonitor.SetTime(startTime) admissionReview := ws.bodyToAdmissionReview(r, rw) if admissionReview == nil { @@ -246,7 +243,7 @@ func (ws *WebhookServer) handlerFunc(handler func(request *v1beta1.AdmissionRequ admissionReview.Response = handler(request) writeResponse(rw, admissionReview) - logger.V(4).Info("request processed", "processingTime", time.Since(startTime).String()) + logger.V(3).Info("admission review request processed", "time", time.Since(startTime).String()) return } @@ -334,10 +331,8 @@ func (ws *WebhookServer) ResourceMutation(request *v1beta1.AdmissionRequest) *v1 var patches []byte patchedResource := request.Object.Raw + // MUTATION if ws.supportMutateValidate { - // MUTATION - // mutation failure should not block the resource creation - // any mutation failure is reported as the violation if resource.GetDeletionTimestamp() == nil { patches = ws.HandleMutation(request, resource, mutatePolicies, ctx, userRequestInfo) logger.V(6).Info("", "generated patches", string(patches)) @@ -346,38 +341,15 @@ func (ws *WebhookServer) ResourceMutation(request *v1beta1.AdmissionRequest) *v1 patchedResource = processResourceWithPatches(patches, request.Object.Raw, logger) logger.V(6).Info("", "patchedResource", string(patchedResource)) } - - if ws.resourceWebhookWatcher != nil && ws.resourceWebhookWatcher.RunValidationInMutatingWebhook == "true" { - // push admission request to audit handler, this won't block the admission request - ws.auditHandler.Add(request.DeepCopy()) - - // VALIDATION - ok, msg := HandleValidation(request, validatePolicies, nil, ctx, userRequestInfo, ws.statusListener, ws.eventGen, ws.prGenerator, ws.log, ws.configHandler, ws.resCache) - if !ok { - logger.Info("admission request denied") - return &v1beta1.AdmissionResponse{ - Allowed: false, - Result: &metav1.Status{ - Status: "Failure", - Message: msg, - }, - } - } - } } else { - logger.Info("mutate and validate rules are not supported prior to Kubernetes 1.14.0") + logger.Info("mutate rules are not supported prior to Kubernetes 1.14.0") } // GENERATE - // Only applied during resource creation and update - // Success -> Generate Request CR created successfully - // Failed -> Failed to create Generate Request CR - if request.Operation == v1beta1.Create || request.Operation == v1beta1.Update { go ws.HandleGenerate(request.DeepCopy(), generatePolicies, ctx, userRequestInfo, ws.configHandler) } - // Successful processing of mutation & validation rules in policy patchType := v1beta1.PatchTypeJSONPatch return &v1beta1.AdmissionResponse{ Allowed: true, @@ -387,7 +359,6 @@ func (ws *WebhookServer) ResourceMutation(request *v1beta1.AdmissionRequest) *v1 Patch: patches, PatchType: &patchType, } - } func (ws *WebhookServer) resourceValidation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { @@ -511,21 +482,19 @@ func (ws *WebhookServer) RunAsync(stopCh <-chan struct{}) { logger.Error(err, "failed to listen to requests") } }() - logger.Info("starting") - // verifies if the admission control is enabled and active - // resync: 60 seconds - // deadline: 60 seconds (send request) - // max deadline: deadline*3 (set the deployment annotation as false) - go ws.lastReqTime.Run(ws.pLister, ws.eventGen, ws.client, checker.DefaultResync, checker.DefaultDeadline, stopCh) + logger.Info("starting service") + + go ws.webhookMonitor.Run(ws.webhookRegister, ws.eventGen, ws.client, stopCh) } // Stop TLS server and returns control after the server is shut down func (ws *WebhookServer) Stop(ctx context.Context) { logger := ws.log - // cleanUp - // remove the static webhookconfigurations - go ws.webhookRegistrationClient.RemoveWebhookConfigurations(ws.cleanUp) + + // remove the static webhook configurations + go ws.webhookRegister.Remove(ws.cleanUp) + // shutdown http.Server with context timeout err := ws.server.Shutdown(ctx) if err != nil {