diff --git a/cmd/initContainer/main.go b/cmd/initContainer/main.go index 4860f4ee0a..7e4f4e5af5 100644 --- a/cmd/initContainer/main.go +++ b/cmd/initContainer/main.go @@ -46,6 +46,8 @@ func main() { requests := []request{ // Resource + request{validatingWebhookConfigKind, config.ValidatingWebhookConfigurationName}, + request{validatingWebhookConfigKind, config.ValidatingWebhookConfigurationDebugName}, request{mutatingWebhookConfigKind, config.MutatingWebhookConfigurationName}, request{mutatingWebhookConfigKind, config.MutatingWebhookConfigurationDebugName}, // Policy diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index 61013df4fb..49aed1eb58 100644 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -27,9 +27,12 @@ import ( ) var ( - kubeconfig string - serverIP string - webhookTimeout int + kubeconfig string + serverIP string + cpu bool + memory bool + webhookTimeout int + runValidationInMutatingWebhook string //TODO: this has been added to backward support command line arguments // will be removed in future and the configuration will be set only via configmaps filterK8Resources string @@ -40,7 +43,6 @@ var ( func main() { defer glog.Flush() version.PrintVersionInfo() - // cleanUp Channel cleanUp := make(chan struct{}) // handle os signals @@ -103,7 +105,9 @@ func main() { rWebhookWatcher := webhookconfig.NewResourceWebhookRegister( lastReqTime, kubeInformer.Admissionregistration().V1beta1().MutatingWebhookConfigurations(), + kubeInformer.Admissionregistration().V1beta1().ValidatingWebhookConfigurations(), webhookRegistrationClient, + runValidationInMutatingWebhook, ) // KYVERNO CRD INFORMER @@ -265,6 +269,8 @@ func init() { flag.IntVar(&webhookTimeout, "webhooktimeout", 3, "timeout for webhook configurations") flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") flag.StringVar(&serverIP, "serverIP", "", "IP address where Kyverno controller runs. Only required if out-of-cluster.") + flag.StringVar(&runValidationInMutatingWebhook, "runValidationInMutatingWebhook", "", "Validation will also be done using the mutation webhook, set to 'true' to enable. Older kubernetes versions do not work properly when a validation webhook is registered.") + // Generate CSR with CN as FQDN due to https://github.com/nirmata/kyverno/issues/542 flag.BoolVar(&fqdncn, "fqdn-as-cn", false, "use FQDN as Common Name in CSR") config.LogDefaultFlags() diff --git a/pkg/config/config.go b/pkg/config/config.go index f553fa8122..f1871c4659 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,9 +17,9 @@ const ( MutatingWebhookConfigurationDebugName = "kyverno-resource-mutating-webhook-cfg-debug" MutatingWebhookName = "nirmata.kyverno.resource.mutating-webhook" - // ValidatingWebhookConfigurationName = "kyverno-validating-webhook-cfg" - // ValidatingWebhookConfigurationDebug = "kyverno-validating-webhook-cfg-debug" - // ValidatingWebhookName = "nirmata.kyverno.policy-validating-webhook" + ValidatingWebhookConfigurationName = "kyverno-validating-webhook-cfg" + ValidatingWebhookConfigurationDebugName = "kyverno-validating-webhook-cfg-debug" + ValidatingWebhookName = "nirmata.kyverno.policy-validating-webhook" VerifyMutatingWebhookConfigurationName = "kyverno-verify-mutating-webhook-cfg" VerifyMutatingWebhookConfigurationDebugName = "kyverno-verify-mutating-webhook-cfg-debug" diff --git a/pkg/webhookconfig/registration.go b/pkg/webhookconfig/registration.go index 9e0afec396..e0d1008d99 100644 --- a/pkg/webhookconfig/registration.go +++ b/pkg/webhookconfig/registration.go @@ -84,7 +84,7 @@ func (wrc *WebhookRegistrationClient) RemoveWebhookConfigurations(cleanUp chan<- //CreateResourceMutatingWebhookConfiguration create a Mutatingwebhookconfiguration resource for all resource type // used to forward request to kyverno webhooks to apply policeis -// Mutationg webhook is be used for Mutating & Validating purpose +// Mutationg webhook is be used for Mutating purpose func (wrc *WebhookRegistrationClient) CreateResourceMutatingWebhookConfiguration() error { var caData []byte var config *admregapi.MutatingWebhookConfiguration @@ -99,7 +99,7 @@ func (wrc *WebhookRegistrationClient) CreateResourceMutatingWebhookConfiguration if wrc.serverIP != "" { // debug mode // clientConfig - URL - config = wrc.contructDebugMutatingWebhookConfig(caData) + config = wrc.constructDebugMutatingWebhookConfig(caData) } else { // clientConfig - service config = wrc.constructMutatingWebhookConfig(caData) @@ -116,6 +116,35 @@ func (wrc *WebhookRegistrationClient) CreateResourceMutatingWebhookConfiguration return nil } +func (wrc *WebhookRegistrationClient) CreateResourceValidatingWebhookConfiguration() error { + var caData []byte + var config *admregapi.ValidatingWebhookConfiguration + + if caData = wrc.readCaData(); caData == nil { + return errors.New("Unable to extract CA data from configuration") + } + // if serverIP is specified we assume its debug mode + if wrc.serverIP != "" { + // debug mode + // clientConfig - URL + config = wrc.constructDebugValidatingWebhookConfig(caData) + } else { + // clientConfig - service + config = wrc.constructValidatingWebhookConfig(caData) + } + + _, err := wrc.client.CreateResource(ValidatingWebhookConfigurationKind, "", *config, false) + if errorsapi.IsAlreadyExists(err) { + glog.V(4).Infof("resource validating webhook configuration %s, already exists. not creating one", config.Name) + return nil + } + if err != nil { + glog.V(4).Infof("failed to create resource validating webhook configuration %s: %v", config.Name, err) + return err + } + return nil +} + //registerPolicyValidatingWebhookConfiguration create a Validating webhook configuration for Policy CRD func (wrc *WebhookRegistrationClient) createPolicyValidatingWebhookConfiguration() error { var caData []byte @@ -219,9 +248,10 @@ func (wrc *WebhookRegistrationClient) removeWebhookConfigurations() { var wg sync.WaitGroup - wg.Add(4) + wg.Add(5) // mutating and validating webhook configuration for Kubernetes resources go wrc.removeResourceMutatingWebhookConfiguration(&wg) + go wrc.removeResourceValidatingWebhookConfiguration(&wg) // mutating and validating webhook configurtion for Policy CRD resource go wrc.removePolicyMutatingWebhookConfiguration(&wg) go wrc.removePolicyValidatingWebhookConfiguration(&wg) @@ -238,6 +268,10 @@ func (wrc *WebhookRegistrationClient) removeResourceMutatingWebhookConfiguration defer wg.Done() wrc.RemoveResourceMutatingWebhookConfiguration() } +func (wrc *WebhookRegistrationClient) removeResourceValidatingWebhookConfiguration(wg *sync.WaitGroup) { + defer wg.Done() + wrc.RemoveResourceValidatingWebhookConfiguration() +} // delete policy mutating webhookconfigurations // handle wait group diff --git a/pkg/webhookconfig/resource.go b/pkg/webhookconfig/resource.go index 8a32a8863e..090c33636a 100644 --- a/pkg/webhookconfig/resource.go +++ b/pkg/webhookconfig/resource.go @@ -10,7 +10,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func (wrc *WebhookRegistrationClient) contructDebugMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { +func (wrc *WebhookRegistrationClient) constructDebugMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.MutatingWebhookServicePath) glog.V(4).Infof("Debug MutatingWebhookConfig is registered with url %s\n", url) @@ -83,3 +83,73 @@ func (wrc *WebhookRegistrationClient) RemoveResourceMutatingWebhookConfiguration glog.V(4).Infof("deleted resource webhook configuration %s", configName) return nil } + +func (wrc *WebhookRegistrationClient) constructDebugValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration { + url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.ValidatingWebhookServicePath) + glog.V(4).Infof("Debug ValidatingWebhookConfig is registered with url %s\n", url) + + return &admregapi.ValidatingWebhookConfiguration{ + ObjectMeta: v1.ObjectMeta{ + Name: config.ValidatingWebhookConfigurationDebugName, + }, + Webhooks: []admregapi.Webhook{ + generateDebugWebhook( + config.ValidatingWebhookName, + url, + caData, + true, + wrc.timeoutSeconds, + "*/*", + "*", + "*", + []admregapi.OperationType{admregapi.Create, admregapi.Update}, + ), + }, + } +} + +func (wrc *WebhookRegistrationClient) constructValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration { + return &admregapi.ValidatingWebhookConfiguration{ + ObjectMeta: v1.ObjectMeta{ + Name: config.ValidatingWebhookConfigurationName, + OwnerReferences: []v1.OwnerReference{ + wrc.constructOwner(), + }, + }, + Webhooks: []admregapi.Webhook{ + generateWebhook( + config.ValidatingWebhookName, + config.ValidatingWebhookServicePath, + caData, + false, + wrc.timeoutSeconds, + "*/*", + "*", + "*", + []admregapi.OperationType{admregapi.Create, admregapi.Update}, + ), + }, + } +} + +func (wrc *WebhookRegistrationClient) GetResourceValidatingWebhookConfigName() string { + if wrc.serverIP != "" { + return config.ValidatingWebhookConfigurationDebugName + } + return config.ValidatingWebhookConfigurationName +} + +func (wrc *WebhookRegistrationClient) RemoveResourceValidatingWebhookConfiguration() error { + configName := wrc.GetResourceValidatingWebhookConfigName() + err := wrc.client.DeleteResource(ValidatingWebhookConfigurationKind, "", configName, false) + if errors.IsNotFound(err) { + glog.V(4).Infof("resource webhook configuration %s does not exits, so not deleting", configName) + return nil + } + if err != nil { + glog.V(4).Infof("failed to delete resource webhook configuration %s: %v", configName, err) + return err + } + glog.V(4).Infof("deleted resource webhook configuration %s", configName) + return nil +} diff --git a/pkg/webhookconfig/rwebhookregister.go b/pkg/webhookconfig/rwebhookregister.go index 17c321f426..9a1f5d389b 100644 --- a/pkg/webhookconfig/rwebhookregister.go +++ b/pkg/webhookconfig/rwebhookregister.go @@ -16,22 +16,30 @@ type ResourceWebhookRegister struct { pendingCreation *abool.AtomicBool LastReqTime *checker.LastReqTime mwebhookconfigSynced cache.InformerSynced + vwebhookconfigSynced cache.InformerSynced // list/get mutatingwebhookconfigurations - mWebhookConfigLister mconfiglister.MutatingWebhookConfigurationLister - webhookRegistrationClient *WebhookRegistrationClient + mWebhookConfigLister mconfiglister.MutatingWebhookConfigurationLister + vWebhookConfigLister mconfiglister.ValidatingWebhookConfigurationLister + webhookRegistrationClient *WebhookRegistrationClient + RunValidationInMutatingWebhook string } func NewResourceWebhookRegister( lastReqTime *checker.LastReqTime, mconfigwebhookinformer mconfiginformer.MutatingWebhookConfigurationInformer, + vconfigwebhookinformer mconfiginformer.ValidatingWebhookConfigurationInformer, webhookRegistrationClient *WebhookRegistrationClient, + runValidationInMutatingWebhook string, ) *ResourceWebhookRegister { return &ResourceWebhookRegister{ - pendingCreation: abool.New(), - LastReqTime: lastReqTime, - mwebhookconfigSynced: mconfigwebhookinformer.Informer().HasSynced, - mWebhookConfigLister: mconfigwebhookinformer.Lister(), - webhookRegistrationClient: webhookRegistrationClient, + pendingCreation: abool.New(), + LastReqTime: lastReqTime, + mwebhookconfigSynced: mconfigwebhookinformer.Informer().HasSynced, + mWebhookConfigLister: mconfigwebhookinformer.Lister(), + vwebhookconfigSynced: vconfigwebhookinformer.Informer().HasSynced, + vWebhookConfigLister: vconfigwebhookinformer.Lister(), + webhookRegistrationClient: webhookRegistrationClient, + RunValidationInMutatingWebhook: runValidationInMutatingWebhook, } } @@ -42,60 +50,84 @@ func (rww *ResourceWebhookRegister) RegisterResourceWebhook() { return } - // check cache - configName := rww.webhookRegistrationClient.GetResourceMutatingWebhookConfigName() - // exsitence of config is all that matters; if error occurs, creates webhook anyway - // errors of webhook creation are handled separately - config, _ := rww.mWebhookConfigLister.Get(configName) - if config != nil { - glog.V(4).Info("mutating webhoook configuration already exists, skip the request") - return - } - - createWebhook := func() { - rww.pendingCreation.Set() - err := rww.webhookRegistrationClient.CreateResourceMutatingWebhookConfiguration() - rww.pendingCreation.UnSet() - - if err != nil { - glog.Errorf("failed to create resource mutating webhook configuration: %v, re-queue creation request", err) - rww.RegisterResourceWebhook() - return - } - glog.V(3).Info("Successfully created mutating webhook configuration for resources") - } - timeDiff := time.Since(rww.LastReqTime.Time()) if timeDiff < checker.DefaultDeadline { glog.V(3).Info("Verified webhook status, creating webhook configuration") - go createWebhook() + go func() { + mutatingConfigName := rww.webhookRegistrationClient.GetResourceMutatingWebhookConfigName() + mutatingConfig, _ := rww.mWebhookConfigLister.Get(mutatingConfigName) + if mutatingConfig != nil { + glog.V(4).Info("mutating webhoook configuration already exists") + } else { + rww.pendingCreation.Set() + err1 := rww.webhookRegistrationClient.CreateResourceMutatingWebhookConfiguration() + rww.pendingCreation.UnSet() + if err1 != nil { + glog.Errorf("failed to create resource mutating webhook configuration: %v, re-queue creation request", err1) + rww.RegisterResourceWebhook() + return + } + glog.V(3).Info("Successfully created mutating webhook configuration for resources") + } + + if rww.RunValidationInMutatingWebhook != "true" { + validatingConfigName := rww.webhookRegistrationClient.GetResourceValidatingWebhookConfigName() + validatingConfig, _ := rww.vWebhookConfigLister.Get(validatingConfigName) + if validatingConfig != nil { + glog.V(4).Info("validating webhoook configuration already exists") + } else { + rww.pendingCreation.Set() + err2 := rww.webhookRegistrationClient.CreateResourceValidatingWebhookConfiguration() + rww.pendingCreation.UnSet() + if err2 != nil { + glog.Errorf("failed to create resource validating webhook configuration: %v, re-queue creation request", err2) + rww.RegisterResourceWebhook() + return + } + glog.V(3).Info("Successfully created validating webhook configuration for resources") + } + } + }() } } func (rww *ResourceWebhookRegister) Run(stopCh <-chan struct{}) { // wait for cache to populate first time - if !cache.WaitForCacheSync(stopCh, rww.mwebhookconfigSynced) { + if !cache.WaitForCacheSync(stopCh, rww.mwebhookconfigSynced, rww.vwebhookconfigSynced) { glog.Error("configuration: failed to sync webhook informer cache") } + } func (rww *ResourceWebhookRegister) RemoveResourceWebhookConfiguration() error { - var err error - // check informer cache - configName := rww.webhookRegistrationClient.GetResourceMutatingWebhookConfigName() - config, err := rww.mWebhookConfigLister.Get(configName) + mutatingConfigName := rww.webhookRegistrationClient.GetResourceMutatingWebhookConfigName() + mutatingConfig, err := rww.mWebhookConfigLister.Get(mutatingConfigName) if err != nil { glog.V(4).Infof("failed to list mutating webhook config: %v", err) return err } - if config == nil { - // as no resource is found - return nil + if mutatingConfig != nil { + err = rww.webhookRegistrationClient.RemoveResourceMutatingWebhookConfiguration() + if err != nil { + return err + } + glog.V(3).Info("removed mutating resource webhook configuration") } - err = rww.webhookRegistrationClient.RemoveResourceMutatingWebhookConfiguration() - if err != nil { - return err + + if rww.RunValidationInMutatingWebhook != "true" { + validatingConfigName := rww.webhookRegistrationClient.GetResourceValidatingWebhookConfigName() + validatingConfig, err := rww.vWebhookConfigLister.Get(validatingConfigName) + if err != nil { + glog.V(4).Infof("failed to list validating webhook config: %v", err) + return err + } + if validatingConfig != nil { + err = rww.webhookRegistrationClient.RemoveResourceValidatingWebhookConfiguration() + if err != nil { + return err + } + glog.V(3).Info("removed validating resource webhook configuration") + } } - glog.V(3).Info("removed resource webhook configuration") return nil } diff --git a/pkg/webhooks/common.go b/pkg/webhooks/common.go index 22cbc2084d..611e81cd2e 100644 --- a/pkg/webhooks/common.go +++ b/pkg/webhooks/common.go @@ -6,8 +6,8 @@ import ( "github.com/golang/glog" kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1" - engineutils "github.com/nirmata/kyverno/pkg/engine/utils" "github.com/nirmata/kyverno/pkg/engine/response" + engineutils "github.com/nirmata/kyverno/pkg/engine/utils" "k8s.io/api/admission/v1beta1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -119,6 +119,9 @@ func extractResources(newRaw []byte, request *v1beta1.AdmissionRequest) (unstruc var emptyResource unstructured.Unstructured // New Resource + if newRaw == nil { + newRaw = request.Object.Raw + } if newRaw == nil { return emptyResource, emptyResource, fmt.Errorf("new resource is not defined") } diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index 1e7c1eb8d8..81b974a9ad 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -123,6 +123,7 @@ func NewWebhookServer( } mux := http.NewServeMux() mux.HandleFunc(config.MutatingWebhookServicePath, ws.serve) + mux.HandleFunc(config.ValidatingWebhookServicePath, ws.serve) mux.HandleFunc(config.VerifyMutatingWebhookServicePath, ws.serve) mux.HandleFunc(config.PolicyValidatingWebhookServicePath, ws.serve) mux.HandleFunc(config.PolicyMutatingWebhookServicePath, ws.serve) @@ -164,7 +165,11 @@ func (ws *WebhookServer) serve(w http.ResponseWriter, r *http.Request) { admissionReview.Response = ws.handleVerifyRequest(request) case config.MutatingWebhookServicePath: if !ws.configHandler.ToFilter(request.Kind.Kind, request.Namespace, request.Name) { - admissionReview.Response = ws.handleAdmissionRequest(request) + admissionReview.Response = ws.handleMutateAdmissionRequest(request) + } + case config.ValidatingWebhookServicePath: + if !ws.configHandler.ToFilter(request.Kind.Kind, request.Namespace, request.Name) { + admissionReview.Response = ws.handleValidateAdmissionRequest(request) } case config.PolicyValidatingWebhookServicePath: if !ws.configHandler.ToFilter(request.Kind.Kind, request.Namespace, request.Name) { @@ -189,7 +194,7 @@ func (ws *WebhookServer) serve(w http.ResponseWriter, r *http.Request) { } } -func (ws *WebhookServer) handleAdmissionRequest(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { +func (ws *WebhookServer) handleMutateAdmissionRequest(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { policies, err := ws.pMetaStore.LookUp(request.Kind.Kind, request.Namespace) if err != nil { // Unable to connect to policy Lister to access policies @@ -250,16 +255,18 @@ func (ws *WebhookServer) handleAdmissionRequest(request *v1beta1.AdmissionReques // patch the resource with patches before handling validation rules patchedResource := processResourceWithPatches(patches, request.Object.Raw) - // VALIDATION - ok, msg = ws.HandleValidation(request, policies, patchedResource, roles, clusterRoles) - if !ok { - glog.V(4).Infof("Deny admission request: %v/%s/%s", request.Kind, request.Namespace, request.Name) - return &v1beta1.AdmissionResponse{ - Allowed: false, - Result: &metav1.Status{ - Status: "Failure", - Message: msg, - }, + if ws.resourceWebhookWatcher != nil && ws.resourceWebhookWatcher.RunValidationInMutatingWebhook == "true" { + // VALIDATION + ok, msg = ws.HandleValidation(request, policies, patchedResource, roles, clusterRoles) + if !ok { + glog.V(4).Infof("Deny admission request: %v/%s/%s", request.Kind, request.Namespace, request.Name) + return &v1beta1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Status: "Failure", + Message: msg, + }, + } } } @@ -292,6 +299,49 @@ func (ws *WebhookServer) handleAdmissionRequest(request *v1beta1.AdmissionReques } } +func (ws *WebhookServer) handleValidateAdmissionRequest(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { + policies, err := ws.pMetaStore.LookUp(request.Kind.Kind, request.Namespace) + if err != nil { + // Unable to connect to policy Lister to access policies + glog.Errorf("Unable to connect to policy controller to access policies. Policies are NOT being applied: %v", err) + return &v1beta1.AdmissionResponse{Allowed: true} + } + + var roles, clusterRoles []string + + // getRoleRef only if policy has roles/clusterroles defined + startTime := time.Now() + if containRBACinfo(policies) { + roles, clusterRoles, err = userinfo.GetRoleRef(ws.rbLister, ws.crbLister, request) + if err != nil { + // TODO(shuting): continue apply policy if error getting roleRef? + glog.Errorf("Unable to get rbac information for request Kind=%s, Namespace=%s Name=%s UID=%s patchOperation=%s: %v", + request.Kind.Kind, request.Namespace, request.Name, request.UID, request.Operation, err) + } + } + glog.V(4).Infof("Time: webhook GetRoleRef %v", time.Since(startTime)) + + // VALIDATION + ok, msg := ws.HandleValidation(request, policies, nil, roles, clusterRoles) + if !ok { + glog.V(4).Infof("Deny admission request: %v/%s/%s", request.Kind, request.Namespace, request.Name) + return &v1beta1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Status: "Failure", + Message: msg, + }, + } + } + + return &v1beta1.AdmissionResponse{ + Allowed: true, + Result: &metav1.Status{ + Status: "Success", + }, + } +} + // RunAsync TLS server in separate thread and returns control immediately func (ws *WebhookServer) RunAsync(stopCh <-chan struct{}) { if !cache.WaitForCacheSync(stopCh, ws.pSynced, ws.rbSynced, ws.crbSynced) {