From 8dc6b06d79429841e6aa43907060c9c20cb6ef36 Mon Sep 17 00:00:00 2001 From: shravan Date: Sat, 11 Jan 2020 18:33:11 +0530 Subject: [PATCH 01/14] resolving merge conflicts --- cmd/initContainer/main.go | 2 + cmd/kyverno/main.go | 14 ++- pkg/config/config.go | 6 +- pkg/webhookconfig/registration.go | 40 ++++++++- pkg/webhookconfig/resource.go | 72 +++++++++++++++- pkg/webhookconfig/rwebhookregister.go | 118 ++++++++++++++++---------- pkg/webhooks/common.go | 5 +- pkg/webhooks/server.go | 74 +++++++++++++--- 8 files changed, 264 insertions(+), 67 deletions(-) diff --git a/cmd/initContainer/main.go b/cmd/initContainer/main.go index 4860f4ee0a..7e4f4e5af5 100644 --- a/cmd/initContainer/main.go +++ b/cmd/initContainer/main.go @@ -46,6 +46,8 @@ func main() { requests := []request{ // Resource + request{validatingWebhookConfigKind, config.ValidatingWebhookConfigurationName}, + request{validatingWebhookConfigKind, config.ValidatingWebhookConfigurationDebugName}, request{mutatingWebhookConfigKind, config.MutatingWebhookConfigurationName}, request{mutatingWebhookConfigKind, config.MutatingWebhookConfigurationDebugName}, // Policy diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index 61013df4fb..49aed1eb58 100644 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -27,9 +27,12 @@ import ( ) var ( - kubeconfig string - serverIP string - webhookTimeout int + kubeconfig string + serverIP string + cpu bool + memory bool + webhookTimeout int + runValidationInMutatingWebhook string //TODO: this has been added to backward support command line arguments // will be removed in future and the configuration will be set only via configmaps filterK8Resources string @@ -40,7 +43,6 @@ var ( func main() { defer glog.Flush() version.PrintVersionInfo() - // cleanUp Channel cleanUp := make(chan struct{}) // handle os signals @@ -103,7 +105,9 @@ func main() { rWebhookWatcher := webhookconfig.NewResourceWebhookRegister( lastReqTime, kubeInformer.Admissionregistration().V1beta1().MutatingWebhookConfigurations(), + kubeInformer.Admissionregistration().V1beta1().ValidatingWebhookConfigurations(), webhookRegistrationClient, + runValidationInMutatingWebhook, ) // KYVERNO CRD INFORMER @@ -265,6 +269,8 @@ func init() { flag.IntVar(&webhookTimeout, "webhooktimeout", 3, "timeout for webhook configurations") flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") flag.StringVar(&serverIP, "serverIP", "", "IP address where Kyverno controller runs. Only required if out-of-cluster.") + flag.StringVar(&runValidationInMutatingWebhook, "runValidationInMutatingWebhook", "", "Validation will also be done using the mutation webhook, set to 'true' to enable. Older kubernetes versions do not work properly when a validation webhook is registered.") + // Generate CSR with CN as FQDN due to https://github.com/nirmata/kyverno/issues/542 flag.BoolVar(&fqdncn, "fqdn-as-cn", false, "use FQDN as Common Name in CSR") config.LogDefaultFlags() diff --git a/pkg/config/config.go b/pkg/config/config.go index f553fa8122..f1871c4659 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,9 +17,9 @@ const ( MutatingWebhookConfigurationDebugName = "kyverno-resource-mutating-webhook-cfg-debug" MutatingWebhookName = "nirmata.kyverno.resource.mutating-webhook" - // ValidatingWebhookConfigurationName = "kyverno-validating-webhook-cfg" - // ValidatingWebhookConfigurationDebug = "kyverno-validating-webhook-cfg-debug" - // ValidatingWebhookName = "nirmata.kyverno.policy-validating-webhook" + ValidatingWebhookConfigurationName = "kyverno-validating-webhook-cfg" + ValidatingWebhookConfigurationDebugName = "kyverno-validating-webhook-cfg-debug" + ValidatingWebhookName = "nirmata.kyverno.policy-validating-webhook" VerifyMutatingWebhookConfigurationName = "kyverno-verify-mutating-webhook-cfg" VerifyMutatingWebhookConfigurationDebugName = "kyverno-verify-mutating-webhook-cfg-debug" diff --git a/pkg/webhookconfig/registration.go b/pkg/webhookconfig/registration.go index 9e0afec396..e0d1008d99 100644 --- a/pkg/webhookconfig/registration.go +++ b/pkg/webhookconfig/registration.go @@ -84,7 +84,7 @@ func (wrc *WebhookRegistrationClient) RemoveWebhookConfigurations(cleanUp chan<- //CreateResourceMutatingWebhookConfiguration create a Mutatingwebhookconfiguration resource for all resource type // used to forward request to kyverno webhooks to apply policeis -// Mutationg webhook is be used for Mutating & Validating purpose +// Mutationg webhook is be used for Mutating purpose func (wrc *WebhookRegistrationClient) CreateResourceMutatingWebhookConfiguration() error { var caData []byte var config *admregapi.MutatingWebhookConfiguration @@ -99,7 +99,7 @@ func (wrc *WebhookRegistrationClient) CreateResourceMutatingWebhookConfiguration if wrc.serverIP != "" { // debug mode // clientConfig - URL - config = wrc.contructDebugMutatingWebhookConfig(caData) + config = wrc.constructDebugMutatingWebhookConfig(caData) } else { // clientConfig - service config = wrc.constructMutatingWebhookConfig(caData) @@ -116,6 +116,35 @@ func (wrc *WebhookRegistrationClient) CreateResourceMutatingWebhookConfiguration return nil } +func (wrc *WebhookRegistrationClient) CreateResourceValidatingWebhookConfiguration() error { + var caData []byte + var config *admregapi.ValidatingWebhookConfiguration + + if caData = wrc.readCaData(); caData == nil { + return errors.New("Unable to extract CA data from configuration") + } + // if serverIP is specified we assume its debug mode + if wrc.serverIP != "" { + // debug mode + // clientConfig - URL + config = wrc.constructDebugValidatingWebhookConfig(caData) + } else { + // clientConfig - service + config = wrc.constructValidatingWebhookConfig(caData) + } + + _, err := wrc.client.CreateResource(ValidatingWebhookConfigurationKind, "", *config, false) + if errorsapi.IsAlreadyExists(err) { + glog.V(4).Infof("resource validating webhook configuration %s, already exists. not creating one", config.Name) + return nil + } + if err != nil { + glog.V(4).Infof("failed to create resource validating webhook configuration %s: %v", config.Name, err) + return err + } + return nil +} + //registerPolicyValidatingWebhookConfiguration create a Validating webhook configuration for Policy CRD func (wrc *WebhookRegistrationClient) createPolicyValidatingWebhookConfiguration() error { var caData []byte @@ -219,9 +248,10 @@ func (wrc *WebhookRegistrationClient) removeWebhookConfigurations() { var wg sync.WaitGroup - wg.Add(4) + wg.Add(5) // mutating and validating webhook configuration for Kubernetes resources go wrc.removeResourceMutatingWebhookConfiguration(&wg) + go wrc.removeResourceValidatingWebhookConfiguration(&wg) // mutating and validating webhook configurtion for Policy CRD resource go wrc.removePolicyMutatingWebhookConfiguration(&wg) go wrc.removePolicyValidatingWebhookConfiguration(&wg) @@ -238,6 +268,10 @@ func (wrc *WebhookRegistrationClient) removeResourceMutatingWebhookConfiguration defer wg.Done() wrc.RemoveResourceMutatingWebhookConfiguration() } +func (wrc *WebhookRegistrationClient) removeResourceValidatingWebhookConfiguration(wg *sync.WaitGroup) { + defer wg.Done() + wrc.RemoveResourceValidatingWebhookConfiguration() +} // delete policy mutating webhookconfigurations // handle wait group diff --git a/pkg/webhookconfig/resource.go b/pkg/webhookconfig/resource.go index 8a32a8863e..090c33636a 100644 --- a/pkg/webhookconfig/resource.go +++ b/pkg/webhookconfig/resource.go @@ -10,7 +10,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func (wrc *WebhookRegistrationClient) contructDebugMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { +func (wrc *WebhookRegistrationClient) constructDebugMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.MutatingWebhookServicePath) glog.V(4).Infof("Debug MutatingWebhookConfig is registered with url %s\n", url) @@ -83,3 +83,73 @@ func (wrc *WebhookRegistrationClient) RemoveResourceMutatingWebhookConfiguration glog.V(4).Infof("deleted resource webhook configuration %s", configName) return nil } + +func (wrc *WebhookRegistrationClient) constructDebugValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration { + url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.ValidatingWebhookServicePath) + glog.V(4).Infof("Debug ValidatingWebhookConfig is registered with url %s\n", url) + + return &admregapi.ValidatingWebhookConfiguration{ + ObjectMeta: v1.ObjectMeta{ + Name: config.ValidatingWebhookConfigurationDebugName, + }, + Webhooks: []admregapi.Webhook{ + generateDebugWebhook( + config.ValidatingWebhookName, + url, + caData, + true, + wrc.timeoutSeconds, + "*/*", + "*", + "*", + []admregapi.OperationType{admregapi.Create, admregapi.Update}, + ), + }, + } +} + +func (wrc *WebhookRegistrationClient) constructValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration { + return &admregapi.ValidatingWebhookConfiguration{ + ObjectMeta: v1.ObjectMeta{ + Name: config.ValidatingWebhookConfigurationName, + OwnerReferences: []v1.OwnerReference{ + wrc.constructOwner(), + }, + }, + Webhooks: []admregapi.Webhook{ + generateWebhook( + config.ValidatingWebhookName, + config.ValidatingWebhookServicePath, + caData, + false, + wrc.timeoutSeconds, + "*/*", + "*", + "*", + []admregapi.OperationType{admregapi.Create, admregapi.Update}, + ), + }, + } +} + +func (wrc *WebhookRegistrationClient) GetResourceValidatingWebhookConfigName() string { + if wrc.serverIP != "" { + return config.ValidatingWebhookConfigurationDebugName + } + return config.ValidatingWebhookConfigurationName +} + +func (wrc *WebhookRegistrationClient) RemoveResourceValidatingWebhookConfiguration() error { + configName := wrc.GetResourceValidatingWebhookConfigName() + err := wrc.client.DeleteResource(ValidatingWebhookConfigurationKind, "", configName, false) + if errors.IsNotFound(err) { + glog.V(4).Infof("resource webhook configuration %s does not exits, so not deleting", configName) + return nil + } + if err != nil { + glog.V(4).Infof("failed to delete resource webhook configuration %s: %v", configName, err) + return err + } + glog.V(4).Infof("deleted resource webhook configuration %s", configName) + return nil +} diff --git a/pkg/webhookconfig/rwebhookregister.go b/pkg/webhookconfig/rwebhookregister.go index 17c321f426..9a1f5d389b 100644 --- a/pkg/webhookconfig/rwebhookregister.go +++ b/pkg/webhookconfig/rwebhookregister.go @@ -16,22 +16,30 @@ type ResourceWebhookRegister struct { pendingCreation *abool.AtomicBool LastReqTime *checker.LastReqTime mwebhookconfigSynced cache.InformerSynced + vwebhookconfigSynced cache.InformerSynced // list/get mutatingwebhookconfigurations - mWebhookConfigLister mconfiglister.MutatingWebhookConfigurationLister - webhookRegistrationClient *WebhookRegistrationClient + mWebhookConfigLister mconfiglister.MutatingWebhookConfigurationLister + vWebhookConfigLister mconfiglister.ValidatingWebhookConfigurationLister + webhookRegistrationClient *WebhookRegistrationClient + RunValidationInMutatingWebhook string } func NewResourceWebhookRegister( lastReqTime *checker.LastReqTime, mconfigwebhookinformer mconfiginformer.MutatingWebhookConfigurationInformer, + vconfigwebhookinformer mconfiginformer.ValidatingWebhookConfigurationInformer, webhookRegistrationClient *WebhookRegistrationClient, + runValidationInMutatingWebhook string, ) *ResourceWebhookRegister { return &ResourceWebhookRegister{ - pendingCreation: abool.New(), - LastReqTime: lastReqTime, - mwebhookconfigSynced: mconfigwebhookinformer.Informer().HasSynced, - mWebhookConfigLister: mconfigwebhookinformer.Lister(), - webhookRegistrationClient: webhookRegistrationClient, + pendingCreation: abool.New(), + LastReqTime: lastReqTime, + mwebhookconfigSynced: mconfigwebhookinformer.Informer().HasSynced, + mWebhookConfigLister: mconfigwebhookinformer.Lister(), + vwebhookconfigSynced: vconfigwebhookinformer.Informer().HasSynced, + vWebhookConfigLister: vconfigwebhookinformer.Lister(), + webhookRegistrationClient: webhookRegistrationClient, + RunValidationInMutatingWebhook: runValidationInMutatingWebhook, } } @@ -42,60 +50,84 @@ func (rww *ResourceWebhookRegister) RegisterResourceWebhook() { return } - // check cache - configName := rww.webhookRegistrationClient.GetResourceMutatingWebhookConfigName() - // exsitence of config is all that matters; if error occurs, creates webhook anyway - // errors of webhook creation are handled separately - config, _ := rww.mWebhookConfigLister.Get(configName) - if config != nil { - glog.V(4).Info("mutating webhoook configuration already exists, skip the request") - return - } - - createWebhook := func() { - rww.pendingCreation.Set() - err := rww.webhookRegistrationClient.CreateResourceMutatingWebhookConfiguration() - rww.pendingCreation.UnSet() - - if err != nil { - glog.Errorf("failed to create resource mutating webhook configuration: %v, re-queue creation request", err) - rww.RegisterResourceWebhook() - return - } - glog.V(3).Info("Successfully created mutating webhook configuration for resources") - } - timeDiff := time.Since(rww.LastReqTime.Time()) if timeDiff < checker.DefaultDeadline { glog.V(3).Info("Verified webhook status, creating webhook configuration") - go createWebhook() + go func() { + mutatingConfigName := rww.webhookRegistrationClient.GetResourceMutatingWebhookConfigName() + mutatingConfig, _ := rww.mWebhookConfigLister.Get(mutatingConfigName) + if mutatingConfig != nil { + glog.V(4).Info("mutating webhoook configuration already exists") + } else { + rww.pendingCreation.Set() + err1 := rww.webhookRegistrationClient.CreateResourceMutatingWebhookConfiguration() + rww.pendingCreation.UnSet() + if err1 != nil { + glog.Errorf("failed to create resource mutating webhook configuration: %v, re-queue creation request", err1) + rww.RegisterResourceWebhook() + return + } + glog.V(3).Info("Successfully created mutating webhook configuration for resources") + } + + if rww.RunValidationInMutatingWebhook != "true" { + validatingConfigName := rww.webhookRegistrationClient.GetResourceValidatingWebhookConfigName() + validatingConfig, _ := rww.vWebhookConfigLister.Get(validatingConfigName) + if validatingConfig != nil { + glog.V(4).Info("validating webhoook configuration already exists") + } else { + rww.pendingCreation.Set() + err2 := rww.webhookRegistrationClient.CreateResourceValidatingWebhookConfiguration() + rww.pendingCreation.UnSet() + if err2 != nil { + glog.Errorf("failed to create resource validating webhook configuration: %v, re-queue creation request", err2) + rww.RegisterResourceWebhook() + return + } + glog.V(3).Info("Successfully created validating webhook configuration for resources") + } + } + }() } } func (rww *ResourceWebhookRegister) Run(stopCh <-chan struct{}) { // wait for cache to populate first time - if !cache.WaitForCacheSync(stopCh, rww.mwebhookconfigSynced) { + if !cache.WaitForCacheSync(stopCh, rww.mwebhookconfigSynced, rww.vwebhookconfigSynced) { glog.Error("configuration: failed to sync webhook informer cache") } + } func (rww *ResourceWebhookRegister) RemoveResourceWebhookConfiguration() error { - var err error - // check informer cache - configName := rww.webhookRegistrationClient.GetResourceMutatingWebhookConfigName() - config, err := rww.mWebhookConfigLister.Get(configName) + mutatingConfigName := rww.webhookRegistrationClient.GetResourceMutatingWebhookConfigName() + mutatingConfig, err := rww.mWebhookConfigLister.Get(mutatingConfigName) if err != nil { glog.V(4).Infof("failed to list mutating webhook config: %v", err) return err } - if config == nil { - // as no resource is found - return nil + if mutatingConfig != nil { + err = rww.webhookRegistrationClient.RemoveResourceMutatingWebhookConfiguration() + if err != nil { + return err + } + glog.V(3).Info("removed mutating resource webhook configuration") } - err = rww.webhookRegistrationClient.RemoveResourceMutatingWebhookConfiguration() - if err != nil { - return err + + if rww.RunValidationInMutatingWebhook != "true" { + validatingConfigName := rww.webhookRegistrationClient.GetResourceValidatingWebhookConfigName() + validatingConfig, err := rww.vWebhookConfigLister.Get(validatingConfigName) + if err != nil { + glog.V(4).Infof("failed to list validating webhook config: %v", err) + return err + } + if validatingConfig != nil { + err = rww.webhookRegistrationClient.RemoveResourceValidatingWebhookConfiguration() + if err != nil { + return err + } + glog.V(3).Info("removed validating resource webhook configuration") + } } - glog.V(3).Info("removed resource webhook configuration") return nil } diff --git a/pkg/webhooks/common.go b/pkg/webhooks/common.go index 22cbc2084d..611e81cd2e 100644 --- a/pkg/webhooks/common.go +++ b/pkg/webhooks/common.go @@ -6,8 +6,8 @@ import ( "github.com/golang/glog" kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1" - engineutils "github.com/nirmata/kyverno/pkg/engine/utils" "github.com/nirmata/kyverno/pkg/engine/response" + engineutils "github.com/nirmata/kyverno/pkg/engine/utils" "k8s.io/api/admission/v1beta1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -119,6 +119,9 @@ func extractResources(newRaw []byte, request *v1beta1.AdmissionRequest) (unstruc var emptyResource unstructured.Unstructured // New Resource + if newRaw == nil { + newRaw = request.Object.Raw + } if newRaw == nil { return emptyResource, emptyResource, fmt.Errorf("new resource is not defined") } diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index 1e7c1eb8d8..81b974a9ad 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -123,6 +123,7 @@ func NewWebhookServer( } mux := http.NewServeMux() mux.HandleFunc(config.MutatingWebhookServicePath, ws.serve) + mux.HandleFunc(config.ValidatingWebhookServicePath, ws.serve) mux.HandleFunc(config.VerifyMutatingWebhookServicePath, ws.serve) mux.HandleFunc(config.PolicyValidatingWebhookServicePath, ws.serve) mux.HandleFunc(config.PolicyMutatingWebhookServicePath, ws.serve) @@ -164,7 +165,11 @@ func (ws *WebhookServer) serve(w http.ResponseWriter, r *http.Request) { admissionReview.Response = ws.handleVerifyRequest(request) case config.MutatingWebhookServicePath: if !ws.configHandler.ToFilter(request.Kind.Kind, request.Namespace, request.Name) { - admissionReview.Response = ws.handleAdmissionRequest(request) + admissionReview.Response = ws.handleMutateAdmissionRequest(request) + } + case config.ValidatingWebhookServicePath: + if !ws.configHandler.ToFilter(request.Kind.Kind, request.Namespace, request.Name) { + admissionReview.Response = ws.handleValidateAdmissionRequest(request) } case config.PolicyValidatingWebhookServicePath: if !ws.configHandler.ToFilter(request.Kind.Kind, request.Namespace, request.Name) { @@ -189,7 +194,7 @@ func (ws *WebhookServer) serve(w http.ResponseWriter, r *http.Request) { } } -func (ws *WebhookServer) handleAdmissionRequest(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { +func (ws *WebhookServer) handleMutateAdmissionRequest(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { policies, err := ws.pMetaStore.LookUp(request.Kind.Kind, request.Namespace) if err != nil { // Unable to connect to policy Lister to access policies @@ -250,16 +255,18 @@ func (ws *WebhookServer) handleAdmissionRequest(request *v1beta1.AdmissionReques // patch the resource with patches before handling validation rules patchedResource := processResourceWithPatches(patches, request.Object.Raw) - // VALIDATION - ok, msg = ws.HandleValidation(request, policies, patchedResource, roles, clusterRoles) - if !ok { - glog.V(4).Infof("Deny admission request: %v/%s/%s", request.Kind, request.Namespace, request.Name) - return &v1beta1.AdmissionResponse{ - Allowed: false, - Result: &metav1.Status{ - Status: "Failure", - Message: msg, - }, + if ws.resourceWebhookWatcher != nil && ws.resourceWebhookWatcher.RunValidationInMutatingWebhook == "true" { + // VALIDATION + ok, msg = ws.HandleValidation(request, policies, patchedResource, roles, clusterRoles) + if !ok { + glog.V(4).Infof("Deny admission request: %v/%s/%s", request.Kind, request.Namespace, request.Name) + return &v1beta1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Status: "Failure", + Message: msg, + }, + } } } @@ -292,6 +299,49 @@ func (ws *WebhookServer) handleAdmissionRequest(request *v1beta1.AdmissionReques } } +func (ws *WebhookServer) handleValidateAdmissionRequest(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { + policies, err := ws.pMetaStore.LookUp(request.Kind.Kind, request.Namespace) + if err != nil { + // Unable to connect to policy Lister to access policies + glog.Errorf("Unable to connect to policy controller to access policies. Policies are NOT being applied: %v", err) + return &v1beta1.AdmissionResponse{Allowed: true} + } + + var roles, clusterRoles []string + + // getRoleRef only if policy has roles/clusterroles defined + startTime := time.Now() + if containRBACinfo(policies) { + roles, clusterRoles, err = userinfo.GetRoleRef(ws.rbLister, ws.crbLister, request) + if err != nil { + // TODO(shuting): continue apply policy if error getting roleRef? + glog.Errorf("Unable to get rbac information for request Kind=%s, Namespace=%s Name=%s UID=%s patchOperation=%s: %v", + request.Kind.Kind, request.Namespace, request.Name, request.UID, request.Operation, err) + } + } + glog.V(4).Infof("Time: webhook GetRoleRef %v", time.Since(startTime)) + + // VALIDATION + ok, msg := ws.HandleValidation(request, policies, nil, roles, clusterRoles) + if !ok { + glog.V(4).Infof("Deny admission request: %v/%s/%s", request.Kind, request.Namespace, request.Name) + return &v1beta1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Status: "Failure", + Message: msg, + }, + } + } + + return &v1beta1.AdmissionResponse{ + Allowed: true, + Result: &metav1.Status{ + Status: "Success", + }, + } +} + // RunAsync TLS server in separate thread and returns control immediately func (ws *WebhookServer) RunAsync(stopCh <-chan struct{}) { if !cache.WaitForCacheSync(stopCh, ws.pSynced, ws.rbSynced, ws.crbSynced) { From 1b417f42dd9567b5bf006fa1277e044183e1501b Mon Sep 17 00:00:00 2001 From: shravan Date: Wed, 15 Jan 2020 20:29:02 +0530 Subject: [PATCH 02/14] changed validating webhook configuration names --- pkg/config/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index f1871c4659..910e8b17e0 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,9 +17,9 @@ const ( MutatingWebhookConfigurationDebugName = "kyverno-resource-mutating-webhook-cfg-debug" MutatingWebhookName = "nirmata.kyverno.resource.mutating-webhook" - ValidatingWebhookConfigurationName = "kyverno-validating-webhook-cfg" - ValidatingWebhookConfigurationDebugName = "kyverno-validating-webhook-cfg-debug" - ValidatingWebhookName = "nirmata.kyverno.policy-validating-webhook" + ValidatingWebhookConfigurationName = "kyverno-resource-validating-webhook-cfg" + ValidatingWebhookConfigurationDebugName = "kyverno-resource-validating-webhook-cfg-debug" + ValidatingWebhookName = "nirmata.kyverno.resource.validating-webhook" VerifyMutatingWebhookConfigurationName = "kyverno-verify-mutating-webhook-cfg" VerifyMutatingWebhookConfigurationDebugName = "kyverno-verify-mutating-webhook-cfg-debug" From 4bd3603e5d656c6bb35831518b9e8a12dc9bded8 Mon Sep 17 00:00:00 2001 From: shravan Date: Fri, 24 Jan 2020 23:25:39 +0530 Subject: [PATCH 03/14] 253 fixing build issues --- pkg/webhooks/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index 743bdf9f83..363b9a0924 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -267,7 +267,7 @@ func (ws *WebhookServer) handleMutateAdmissionRequest(request *v1beta1.Admission // Success -> Generate Request CR created successsfully // Failed -> Failed to create Generate Request CR if request.Operation == v1beta1.Create { - ok, msg = ws.HandleGenerate(request, policies, patchedResource, roles, clusterRoles) + ok, msg := ws.HandleGenerate(request, policies, patchedResource, roles, clusterRoles) if !ok { glog.V(4).Infof("Deny admission request: %v/%s/%s", request.Kind, request.Namespace, request.Name) return &v1beta1.AdmissionResponse{ From 81ea5ba157dc29ad5b76038a2b1f9eb3885fe948 Mon Sep 17 00:00:00 2001 From: shravan Date: Fri, 24 Jan 2020 23:40:05 +0530 Subject: [PATCH 04/14] 253 fixing circle ci issues --- cmd/kyverno/main.go | 2 -- pkg/webhookconfig/registration.go | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index 49aed1eb58..9de0f0c4b3 100644 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -29,8 +29,6 @@ import ( var ( kubeconfig string serverIP string - cpu bool - memory bool webhookTimeout int runValidationInMutatingWebhook string //TODO: this has been added to backward support command line arguments diff --git a/pkg/webhookconfig/registration.go b/pkg/webhookconfig/registration.go index 22ed74a729..123e741822 100644 --- a/pkg/webhookconfig/registration.go +++ b/pkg/webhookconfig/registration.go @@ -272,7 +272,9 @@ func (wrc *WebhookRegistrationClient) removeResourceMutatingWebhookConfiguration } func (wrc *WebhookRegistrationClient) removeResourceValidatingWebhookConfiguration(wg *sync.WaitGroup) { defer wg.Done() - wrc.RemoveResourceValidatingWebhookConfiguration() + if err := wrc.RemoveResourceValidatingWebhookConfiguration(); err != nil { + glog.Error(err) + } } // delete policy mutating webhookconfigurations From 76fa40151ba86b52de1657e49c98a1320f539199 Mon Sep 17 00:00:00 2001 From: Shivkumar Dudhani Date: Tue, 4 Feb 2020 14:55:12 -0800 Subject: [PATCH 05/14] add background processing documentation (#650) * add background processing documentation * update cmd line arg * CR changes --- documentation/installation.md | 2 +- documentation/writing-policies.md | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/documentation/installation.md b/documentation/installation.md index 702c6bc94d..2599787621 100644 --- a/documentation/installation.md +++ b/documentation/installation.md @@ -10,7 +10,7 @@ There are 2 ways to configure the secure communications link between Kyverno and Kyverno can request a CA signed certificate-key pair from `kube-controller-manager`. This method requires that the kube-controller-manager is configured to act as a certificate signer. To verify that this option is enabled for your cluster, check the command-line args for the kube-controller-manager. If `--cluster-signing-cert-file` and `--cluster-signing-key-file` are passed to the controller manager with paths to your CA's key-pair, then you can proceed to install Kyverno using this method. -**Deploying on EKS requires enabling a command-line argument `--fqdncn` in the 'kyverno' container in the deployment, due to a current limitation with the certificates returned by EKS for CSR(bug: https://github.com/awslabs/amazon-eks-ami/issues/341)** +**Deploying on EKS requires enabling a command-line argument `--fqdn-as-cn` in the 'kyverno' container in the deployment, due to a current limitation with the certificates returned by EKS for CSR(bug: https://github.com/awslabs/amazon-eks-ami/issues/341)** To install Kyverno in a cluster that supports certificate signing, run the following command on a host with kubectl `cluster-admin` access: diff --git a/documentation/writing-policies.md b/documentation/writing-policies.md index b81a0de32e..0cb127137c 100644 --- a/documentation/writing-policies.md +++ b/documentation/writing-policies.md @@ -131,6 +131,24 @@ Operators supported: - Equal - NotEqual +# Background processing +Kyverno applies policies in foreground and background mode. +- `foreground`: leverages admission control webhooks to intercept, and apply policies on, API requests for resource changes. +- `background`: policy-controller applies policies on the existing resoruces after configured re-conciliation time. + +A policy is always enabled for `foreground` processing, but `background` processing is configurable using a boolean flag at `{spec.background}`. + +``` +spec: + background: true + rules: + - name: default-deny-ingress +``` +- Unless specified the default value is `true` +- As the userInformation is only avaiable in the incoming api-request, a policy using userInfo filters and variables reffering to `{{request.userInfo}}` can only be processed in foreground mode. +- When a new policy is created, the policy validation will throw an error if using `userInfo` with a policy defined in background mode. + + # Auto generating rules for pod controllers Writing policies on pods helps address all pod creation flows, but results in errors not being reported when a pod controller object is created. Kyverno solves this issue, by automatically generating rules for pod controllers from a rule written for a pod. @@ -140,7 +158,5 @@ Change the annotation `pod-policies.kyverno.io/autogen-controllers` to customize To disable auto-generating rules for pod controllers, set `pod-policies.kyverno.io/autogen-controllers=none`. - - --- *Read Next >> [Validate](/documentation/writing-policies-validate.md)* \ No newline at end of file From 46afbe5f1407781f3bfb14db3030e434976aa0a8 Mon Sep 17 00:00:00 2001 From: Shravan Shetty <58299697+shravanshetty1@users.noreply.github.com> Date: Thu, 6 Feb 2020 08:05:27 +0530 Subject: [PATCH 06/14] #640 - Patch annotation format changes (#660) * 640 tested prototype * 640 fixing tests * 640 requeted changes related to annotation struct * 640 fixes related to previous commit Co-authored-by: Shravan Suresha Shetty <46517941+shravanss@users.noreply.github.com> --- pkg/webhooks/annotations.go | 37 +++++++++++++++++++------------- pkg/webhooks/annotations_test.go | 6 +++--- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/pkg/webhooks/annotations.go b/pkg/webhooks/annotations.go index 21ed21414d..34d3e10468 100644 --- a/pkg/webhooks/annotations.go +++ b/pkg/webhooks/annotations.go @@ -2,6 +2,9 @@ package webhooks import ( "encoding/json" + "strings" + + yamlv2 "gopkg.in/yaml.v2" jsonpatch "github.com/evanphx/json-patch" "github.com/golang/glog" @@ -11,14 +14,9 @@ import ( ) const ( - policyAnnotation = "policies.kyverno.patches" + policyAnnotation = "policies.kyverno.io~1patches" ) -type policyPatch struct { - PolicyName string `json:"policyname"` - RulePatches interface{} `json:"patches"` -} - type rulePatch struct { RuleName string `json:"rulename"` Op string `json:"op"` @@ -31,6 +29,15 @@ type annresponse struct { Value interface{} `json:"value"` } +var operationToPastTense = map[string]string{ + "add": "added", + "remove": "removed", + "replace": "replaced", + "move": "moved", + "copy": "copied", + "test": "tested", +} + func generateAnnotationPatches(engineResponses []response.EngineResponse) []byte { var annotations map[string]string @@ -52,7 +59,7 @@ func generateAnnotationPatches(engineResponses []response.EngineResponse) []byte return nil } - if _, ok := annotations[policyAnnotation]; ok { + if _, ok := annotations[strings.ReplaceAll(policyAnnotation, "~1", "/")]; ok { // create update patch string patchResponse = annresponse{ Op: "replace", @@ -69,7 +76,7 @@ func generateAnnotationPatches(engineResponses []response.EngineResponse) []byte } } else { // insert 'policies.kyverno.patches' entry in annotation map - annotations[policyAnnotation] = string(value) + annotations[strings.ReplaceAll(policyAnnotation, "~1", "/")] = string(value) patchResponse = annresponse{ Op: "add", Path: "/metadata/annotations", @@ -90,31 +97,31 @@ func generateAnnotationPatches(engineResponses []response.EngineResponse) []byte } func annotationFromEngineResponses(engineResponses []response.EngineResponse) []byte { - var policyPatches []policyPatch + var annotationContent = make(map[string]string) for _, engineResponse := range engineResponses { if !engineResponse.IsSuccesful() { glog.V(3).Infof("Policy %s failed, skip preparing annotation\n", engineResponse.PolicyResponse.Policy) continue } - var pp policyPatch rulePatches := annotationFromPolicyResponse(engineResponse.PolicyResponse) if rulePatches == nil { continue } - pp.RulePatches = rulePatches - pp.PolicyName = engineResponse.PolicyResponse.Policy - policyPatches = append(policyPatches, pp) + policyName := engineResponse.PolicyResponse.Policy + for _, rulePatch := range rulePatches { + annotationContent[rulePatch.RuleName+"."+policyName+".kyverno.io"] = operationToPastTense[rulePatch.Op] + " " + rulePatch.Path + } } // return nil if there's no patches // otherwise result = null, len(result) = 4 - if len(policyPatches) == 0 { + if len(annotationContent) == 0 { return nil } - result, _ := json.Marshal(policyPatches) + result, _ := yamlv2.Marshal(annotationContent) return result } diff --git a/pkg/webhooks/annotations_test.go b/pkg/webhooks/annotations_test.go index 4fc0fc07dd..320e347ea3 100644 --- a/pkg/webhooks/annotations_test.go +++ b/pkg/webhooks/annotations_test.go @@ -43,7 +43,7 @@ func Test_empty_annotation(t *testing.T) { engineResponse := newEngineResponse("mutate-container", "default-imagepullpolicy", []string{patchStr}, true, nil) annPatches := generateAnnotationPatches([]response.EngineResponse{engineResponse}) - expectedPatches := `{"op":"add","path":"/metadata/annotations","value":{"policies.kyverno.patches":"[{\"policyname\":\"mutate-container\",\"patches\":[{\"rulename\":\"default-imagepullpolicy\",\"op\":\"replace\",\"path\":\"/spec/containers/0/imagePullPolicy\"}]}]"}}` + expectedPatches := `{"op":"add","path":"/metadata/annotations","value":{"policies.kyverno.io/patches":"default-imagepullpolicy.mutate-container.kyverno.io: replaced /spec/containers/0/imagePullPolicy\n"}}` assert.Assert(t, string(annPatches) == expectedPatches) } @@ -56,7 +56,7 @@ func Test_exist_annotation(t *testing.T) { engineResponse := newEngineResponse("mutate-container", "default-imagepullpolicy", []string{patchStr}, true, annotation) annPatches := generateAnnotationPatches([]response.EngineResponse{engineResponse}) - expectedPatches := `{"op":"add","path":"/metadata/annotations","value":{"policies.kyverno.patches":"[{\"policyname\":\"mutate-container\",\"patches\":[{\"rulename\":\"default-imagepullpolicy\",\"op\":\"replace\",\"path\":\"/spec/containers/0/imagePullPolicy\"}]}]"}}` + expectedPatches := `{"op":"add","path":"/metadata/annotations","value":{"policies.kyverno.io/patches":"default-imagepullpolicy.mutate-container.kyverno.io: replaced /spec/containers/0/imagePullPolicy\n"}}` assert.Assert(t, string(annPatches) == expectedPatches) } @@ -69,7 +69,7 @@ func Test_exist_kyverno_annotation(t *testing.T) { engineResponse := newEngineResponse("mutate-container", "default-imagepullpolicy", []string{patchStr}, true, annotation) annPatches := generateAnnotationPatches([]response.EngineResponse{engineResponse}) - expectedPatches := `{"op":"add","path":"/metadata/annotations","value":{"policies.kyverno.patches":"[{\"policyname\":\"mutate-container\",\"patches\":[{\"rulename\":\"default-imagepullpolicy\",\"op\":\"replace\",\"path\":\"/spec/containers/0/imagePullPolicy\"}]}]"}}` + expectedPatches := `{"op":"add","path":"/metadata/annotations","value":{"policies.kyverno.io/patches":"default-imagepullpolicy.mutate-container.kyverno.io: replaced /spec/containers/0/imagePullPolicy\n"}}` assert.Assert(t, string(annPatches) == expectedPatches) } From 2068da18a874bc2a87c46164ba96e59dcdba069a Mon Sep 17 00:00:00 2001 From: Jim Bugwadia Date: Thu, 6 Feb 2020 00:04:19 -0800 Subject: [PATCH 07/14] update docs for new features --- README.md | 6 +- documentation/testing-policies.md | 51 +------ documentation/writing-policies-autogen.md | 20 +++ documentation/writing-policies-background.md | 20 +++ documentation/writing-policies-generate.md | 3 +- documentation/writing-policies-mutate.md | 2 +- .../writing-policies-preconditions.md | 30 +++++ documentation/writing-policies-variables.md | 35 +++++ documentation/writing-policies.md | 126 +++--------------- 9 files changed, 129 insertions(+), 164 deletions(-) create mode 100644 documentation/writing-policies-autogen.md create mode 100644 documentation/writing-policies-background.md create mode 100644 documentation/writing-policies-preconditions.md create mode 100644 documentation/writing-policies-variables.md diff --git a/README.md b/README.md index 632ac310f2..8e3b82914b 100644 --- a/README.md +++ b/README.md @@ -126,9 +126,11 @@ Refer to a list of curated of ***[sample policies](/samples/README.md)*** that c * [Mutate](documentation/writing-policies-mutate.md) * [Validate](documentation/writing-policies-validate.md) * [Generate](documentation/writing-policies-generate.md) + * [Variables](documentation/writing-policies-variables.md) + * [Preconditions](documentation/writing-policies-preconditions.md) + * [Auto-Generation of Pod Controller Policies](documentation/writing-policies-autogen.md) + * [Background Processing](documentation/writing-policies-background.md) * [Testing Policies](documentation/testing-policies.md) - * [Using kubectl](documentation/testing-policies.md#Test-using-kubectl) - * [Using the Kyverno CLI](documentation/testing-policies.md#Test-using-the-Kyverno-CLI) * [Sample Policies](/samples/README.md) ## License diff --git a/documentation/testing-policies.md b/documentation/testing-policies.md index a877430846..aa24f95836 100644 --- a/documentation/testing-policies.md +++ b/documentation/testing-policies.md @@ -2,9 +2,11 @@ # Testing Policies + The resources definitions for testing are located in [/test](/test) directory. Each test contains a pair of files: one is the resource definition, and the second is the kyverno policy for this definition. ## Test using kubectl + To do this you should [install kyverno to the cluster](/documentation/installation.md). For example, to test the simplest kyverno policy for ConfigMap, create the policy and then the resource itself via kubectl: @@ -19,52 +21,3 @@ Then compare the original resource definition in CM.yaml with the actual one: ````bash kubectl get -f CM.yaml -o yaml ```` - -## Test using the Kyverno CLI - -The Kyverno Command Line Interface (CLI) tool allows writing and testing policies without having to apply local policy changes to a cluster. You can also test policies without a Kubernetes clusters, but results may vary as default values will not be filled in. - -### Building the CLI - -You will need a [Go environment](https://golang.org/doc/install) setup. - -1. Clone the Kyverno repo - -````bash -git clone https://github.com/nirmata/kyverno/ -```` - -2. Build the CLI - -````bash -cd kyverno/cmd/kyverno -go build -```` - -Or, you can directly build and install the CLI using `go get`: - -````bash -go get -u https://github.com/nirmata/kyverno/cmd/kyverno -```` - -### Using the CLI - -The CLI loads default kubeconfig ($HOME/.kube/config) to test policies in Kubernetes cluster. If no kubeconfig is found, the CLI will test policies on raw resources. - -To test a policy using the CLI type: - -`kyverno apply @ @` - -For example: - -```bash -kyverno apply @../../examples/cli/policy-deployment.yaml @../../examples/cli/resources -``` - -To test a policy with the specific kubeconfig: - -```bash -kyverno apply @../../examples/cli/policy-deployment.yaml @../../examples/cli/resources --kubeconfig $PATH_TO_KUBECONFIG_FILE -``` - -In future releases, the CLI will support complete validation and generation of policies. diff --git a/documentation/writing-policies-autogen.md b/documentation/writing-policies-autogen.md new file mode 100644 index 0000000000..baa9a2c314 --- /dev/null +++ b/documentation/writing-policies-autogen.md @@ -0,0 +1,20 @@ +*[documentation](/README.md#documentation) / [Writing Policies](/documentation/writing-policies.md) / Auto-Generation for Pod Controllers* + +# Auto generating rules for pod controllers + +Writing policies on pods helps address all pod creation flows. However, when pod cotrollers are used pod level policies result in errors not being reported when the pod controller object is created. + +Kyverno solves this issue by supporting automatic generation of policy rules for pod controllers from a rule written for a pod. + +This auto-generation behavior is controlled by the `pod-policies.kyverno.io/autogen-controllers` annotation. + +By default, Kyverno inserts an annotation `pod-policies.kyverno.io/autogen-controllers=all`, to generate an additional rule that is applied to pod controllers: DaemonSet, Deployment, Job, StatefulSet. + +You can change the annotation `pod-policies.kyverno.io/autogen-controllers` to customize the target pod controllers for the auto-generated rules. For example, Kyverno generates a rule for a `Deployment` if the annotation of policy is defined as `pod-policies.kyverno.io/autogen-controllers=Deployment`. + +When a `name` or `labelSelector` is specified in the match / exclude block, Kyverno skips generating pod controllers rule as these filters may not be applicable to pod controllers. + +To disable auto-generating rules for pod controllers set `pod-policies.kyverno.io/autogen-controllers` to the value `none`. + +*Read Next >> [Background Processing](/documentation/writing-policies-background.md)* + diff --git a/documentation/writing-policies-background.md b/documentation/writing-policies-background.md new file mode 100644 index 0000000000..b2e3046e11 --- /dev/null +++ b/documentation/writing-policies-background.md @@ -0,0 +1,20 @@ +*[documentation](/README.md#documentation) / [Writing Policies](/documentation/writing-policies.md) / Background Processing* + +# Background processing + +Kyverno applies policies during admission control and to existing resources in the cluster that may have been created before a policy was created. The application of policies to existing resources is referred to as `background` processing. + +Note, that Kyverno does not mutate existing resources, and will only report policy violation for existing resources that do not match mutation, validation, or generation rules. + +A policy is always enabled for processing during admission control. However, policy rules that rely on request information (e.g. `{{request.userInfo}}`) cannot be applied to existing resource in the `background` mode as the user information is not available outside of the admission controller. Hence, these rules must use the boolean flag `{spec.background}` to disable `background` processing. + +``` +spec: + background: true + rules: + - name: default-deny-ingress +``` + +The default value of `background` is `true`. When a policy is created or modified, the policy validation logic will report an error if a rule uses `userInfo` and does not set `background` to `false`. + +*Read Next >> [Testing Policies](/documentation/testing-policies.md)* diff --git a/documentation/writing-policies-generate.md b/documentation/writing-policies-generate.md index 634d69bd6c..837a939546 100644 --- a/documentation/writing-policies-generate.md +++ b/documentation/writing-policies-generate.md @@ -87,5 +87,6 @@ spec: In this example, when the policy is applied, any new namespace will receive a NetworkPolicy based on the specified template that by default denies all inbound and outbound traffic. --- -*Read Next >> [Testing Policies](/documentation/testing-policies.md)* + +*Read Next >> [Variables](/documentation/writing-policies-variables.md)* diff --git a/documentation/writing-policies-mutate.md b/documentation/writing-policies-mutate.md index 7cf5015897..a2eaf48704 100644 --- a/documentation/writing-policies-mutate.md +++ b/documentation/writing-policies-mutate.md @@ -2,7 +2,7 @@ # Mutate Configurations -The ```mutate``` rule contains actions that will be applied to matching resources. A mutate rule can be written as a JSON Patch or as an overlay. +The ```mutate``` rule can be used to add, replace, or delete elements in matching resources. A mutate rule can be written as a JSON Patch or as an overlay. By using a ```patch``` in the (JSONPatch - RFC 6902)[http://jsonpatch.com/] format, you can make precise changes to the resource being created. Using an ```overlay``` is convenient for describing the desired state of the resource. diff --git a/documentation/writing-policies-preconditions.md b/documentation/writing-policies-preconditions.md new file mode 100644 index 0000000000..39da70e482 --- /dev/null +++ b/documentation/writing-policies-preconditions.md @@ -0,0 +1,30 @@ +*[documentation](/README.md#documentation) / [Writing Policies](/documentation/writing-policies.md) / Preconditions* + +# Preconditions + +Preconditions allow controlling policy rule execution based on variable values. + +While `match` & `exclude` conditions allow filtering requests based on resource and user information, `preconditions` can be used to define custom filters for more granular control. + +The following operators are currently supported for preconditon evaluation: +- Equal +- NotEqual + +## Example + +```yaml + - name: generate-owner-role + match: + resources: + kinds: + - Namespace + preconditions: + - key: "{{serviceAccountName}}" + operator: NotEqual + value: "" +``` + +In the above example, the rule is only applied to requests from service accounts i.e. when the `{{serviceAccountName}}` is not empty. + + +*Read Next >> [Auto-Generation for Pod Controllers](/documentation/writing-policies-autogen.md)* diff --git a/documentation/writing-policies-variables.md b/documentation/writing-policies-variables.md new file mode 100644 index 0000000000..a0371e25b4 --- /dev/null +++ b/documentation/writing-policies-variables.md @@ -0,0 +1,35 @@ +*[documentation](/README.md#documentation) / [Writing Policies](/documentation/writing-policies.md) / Variables* + +# Variables + +Sometimes it is necessary to vary the contents of a mutated or generated resource based on request data. To achieve this, variables can be used to reference attributes that are loaded in the rule processing context using a [JMESPATH](http://jmespath.org/) notation. + +The policy engine will substitute any values with the format `{{}}` with the variable value before processing the rule. + +The following data is available for use in context: +- Resource: `{{request.object}}` +- UserInfo: `{{request.userInfo}}` + +## Pre-defined Variables + +Kyverno automatically creates a few useful variables: + +- `serviceAccountName` : the last part of a service account i.e. without the suffix `system:serviceaccount::` and stores the userName. For example, when processing a request from `system:serviceaccount:nirmata:user1` Kyverno will store the value `user1` in the variable `serviceAccountName`. + +- `serviceAccountNamespace` : the `namespace` portion of the serviceAccount. For example, when processing a request from `system:serviceaccount:nirmata:user1` Kyverno will store `nirmata` in the variable `serviceAccountNamespace`. + +## Examples + +1. Reference a resource name (type string) + +`{{request.object.metadata.name}}` + +2. Build name from multiple variables (type string) + +`"ns-owner-{{request.object.metadata.namespace}}-{{request.userInfo.username}}-binding"` + +3. Reference the metadata (type object) + +`{{request.object.metadata}}` + +*Read Next >> [Preconditions](/documentation/writing-policies-preconditions.md)* diff --git a/documentation/writing-policies.md b/documentation/writing-policies.md index 0cb127137c..3bb02ae2e3 100644 --- a/documentation/writing-policies.md +++ b/documentation/writing-policies.md @@ -6,11 +6,17 @@ The following picture shows the structure of a Kyverno Policy: ![KyvernoPolicy](images/Kyverno-Policy-Structure.png) -Each Kyverno policy contains one or more rules. Each rule has a match clause, an optional excludes clause, and a mutate, validate, or generate clause. +Each Kyverno policy contains one or more rules. Each rule has a `match` clause, an optional `exclude` clause, and one of a `mutate`, `validate`, or `generate` clause. + +The match / exclude clauses have the same structure, and can contain the following elements: +* resources: select resources by name, namespaces, kinds, and label selectors. +* subjects: select users and user groups +* roles: select namespaced roles +* clusterroles: select cluster wide roles When Kyverno receives an admission controller request, i.e. a validation or mutation webhook, it first checks to see if the resource and user information matches or should be excluded from processing. If both checks pass, then the rule logic to mutate, validate, or generate resources is applied. -The following YAML provides an example for the match and validate clauses. +The following YAML provides an example for a match clause. ````yaml apiVersion : kyverno.io/v1 @@ -31,11 +37,11 @@ spec : kinds: # Required, list of kinds - Deployment - StatefulSet - name: "mongo*" # Optional, a resource name is optional. Name supports wildcards * and ? - namespaces: # Optional, list of namespaces. Supports wilcards * and ? + name: "mongo*" # Optional, a resource name is optional. Name supports wildcards (* and ?) + namespaces: # Optional, list of namespaces. Supports wildcards (* and ?) - "dev*" - test - selector: # Optional, a resource selector is optional. Selector values support wildcards * and ? + selector: # Optional, a resource selector is optional. Values support wildcards (* and ?) matchLabels: app: mongodb matchExpressions: @@ -47,116 +53,14 @@ spec : # Optional, roles to be matched roles: # Optional, clusterroles to be matched - clusterroles: - # Resources that need to be excluded - exclude: # Optional, resources to be excluded from evaulation - resources: - kinds: - - Daemonsets - name: "*" - namespaces: - - prod - - "kube*" - selector: - matchLabels: - app: mongodb - matchExpressions: - - {key: tier, operator: In, values: [database]} - # Optional, subjects to be excluded - subjects: - # Optional, roles to be excluded - roles: - # Optional, clusterroles to be excluded - clusterroles: - - cluster-admin - - admin - # rule is evaluated if the preconditions are satisfied - # all preconditions are AND/&& operation - preconditions: - - key: name # compares (key operator value) - operator: Equal - value: name # constant "name" == "name" - - key: "{{serviceAccountName}}" # refer to a pre-defined variable serviceAccountName - operator: NotEqual - value: "user1" # if service - # Each rule can contain a single validate, mutate, or generate directive - ... + clusterroles: cluster-admin + + ... + ```` Each rule can validate, mutate, or generate configurations of matching resources. A rule definition can contain only a single **mutate**, **validate**, or **generate** child node. These actions are applied to the resource in described order: mutation, validation and then generation. -# Variables: -Variables can be used to reference attributes that are loaded in the context using a [JMESPATH](http://jmespath.org/) search path. -Format: `{{}}` -Resources available in context: -- Resource: `{{request.object}}` -- UserInfo: `{{request.userInfo}}` - -## Pre-defined Variables -- `serviceAccountName` : the variable removes the suffix system:serviceaccount:: and stores the userName. -Example userName=`system:serviceaccount:nirmata:user1` will store variable value as `user1`. -- `serviceAccountNamespace` : extracts the `namespace` of the serviceAccount. -Example userName=`system:serviceaccount:nirmata:user1` will store variable value as `nirmata`. - -Examples: - -1. Refer to resource name(type string) - -`{{request.object.metadata.name}}` - -2. Build name from multiple variables(type string) - -`"ns-owner-{{request.object.metadata.namespace}}-{{request.userInfo.username}}-binding"` - -3. Refer to metadata struct/object(type object) - -`{{request.object.metadata}}` - -# PreConditions: -Apart from using `match` & `exclude` conditions on resource to filter which resources to apply the rule on, `preconditions` can be used to define custom filters. -```yaml - - name: generate-owner-role - match: - resources: - kinds: - - Namespace - preconditions: - - key: "{{request.userInfo.username}}" - operator: NotEqual - value: "" -``` -In the above example, if the variable `{{request.userInfo.username}}` is blank then we dont apply the rule on resource. - -Operators supported: -- Equal -- NotEqual - -# Background processing -Kyverno applies policies in foreground and background mode. -- `foreground`: leverages admission control webhooks to intercept, and apply policies on, API requests for resource changes. -- `background`: policy-controller applies policies on the existing resoruces after configured re-conciliation time. - -A policy is always enabled for `foreground` processing, but `background` processing is configurable using a boolean flag at `{spec.background}`. - -``` -spec: - background: true - rules: - - name: default-deny-ingress -``` -- Unless specified the default value is `true` -- As the userInformation is only avaiable in the incoming api-request, a policy using userInfo filters and variables reffering to `{{request.userInfo}}` can only be processed in foreground mode. -- When a new policy is created, the policy validation will throw an error if using `userInfo` with a policy defined in background mode. - - -# Auto generating rules for pod controllers -Writing policies on pods helps address all pod creation flows, but results in errors not being reported when a pod controller object is created. Kyverno solves this issue, by automatically generating rules for pod controllers from a rule written for a pod. - -This behavior is controlled by the pod-policies.kyverno.io/autogen-controllers annotation. By default, Kyverno inserts an annotation `pod-policies.kyverno.io/autogen-controllers=all`, to generate an additional rule that is applied to pod controllers: DaemonSet, Deployment, Job, StatefulSet. - -Change the annotation `pod-policies.kyverno.io/autogen-controllers` to customize the applicable pod controllers of the auto-gen rule. For example, Kyverno generates the rule for `Deployment` if the annotation of policy is defined as `pod-policies.kyverno.io/autogen-controllers=Deployment`. If `name` or `labelSelector` is specified in the match / exclude block, Kyverno skips generating pod controllers rule as these filters may not be applicable to pod controllers. - -To disable auto-generating rules for pod controllers, set `pod-policies.kyverno.io/autogen-controllers=none`. --- *Read Next >> [Validate](/documentation/writing-policies-validate.md)* \ No newline at end of file From a93f31ebd53c563090251d77b2176649c3c5188a Mon Sep 17 00:00:00 2001 From: Jim Bugwadia Date: Thu, 6 Feb 2020 00:10:36 -0800 Subject: [PATCH 08/14] update titles --- README.md | 8 ++++---- documentation/writing-policies-generate.md | 4 ++-- documentation/writing-policies-mutate.md | 4 ++-- documentation/writing-policies-validate.md | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 8e3b82914b..55a2381860 100644 --- a/README.md +++ b/README.md @@ -123,10 +123,10 @@ Refer to a list of curated of ***[sample policies](/samples/README.md)*** that c * [Getting Started](documentation/installation.md) * [Writing Policies](documentation/writing-policies.md) - * [Mutate](documentation/writing-policies-mutate.md) - * [Validate](documentation/writing-policies-validate.md) - * [Generate](documentation/writing-policies-generate.md) - * [Variables](documentation/writing-policies-variables.md) + * [Mutate Resources](documentation/writing-policies-mutate.md) + * [Validate Resources](documentation/writing-policies-validate.md) + * [Generate Resources](documentation/writing-policies-generate.md) + * [Variable Substitution](documentation/writing-policies-variables.md) * [Preconditions](documentation/writing-policies-preconditions.md) * [Auto-Generation of Pod Controller Policies](documentation/writing-policies-autogen.md) * [Background Processing](documentation/writing-policies-background.md) diff --git a/documentation/writing-policies-generate.md b/documentation/writing-policies-generate.md index 837a939546..6780e96840 100644 --- a/documentation/writing-policies-generate.md +++ b/documentation/writing-policies-generate.md @@ -1,6 +1,6 @@ -*[documentation](/README.md#documentation) / [Writing Policies](/documentation/writing-policies.md) / Generate* +*[documentation](/README.md#documentation) / [Writing Policies](/documentation/writing-policies.md) / Generate Resources* -# Generate Configurations +# Generate Resources ```generate``` is used to create additional resources when a resource is created. This is useful to create supporting resources, such as role bindings for a new namespace. diff --git a/documentation/writing-policies-mutate.md b/documentation/writing-policies-mutate.md index a2eaf48704..b9a91cb81b 100644 --- a/documentation/writing-policies-mutate.md +++ b/documentation/writing-policies-mutate.md @@ -1,6 +1,6 @@ -*[documentation](/README.md#documentation) / [Writing Policies](/documentation/writing-policies.md) / Mutate* +*[documentation](/README.md#documentation) / [Writing Policies](/documentation/writing-policies.md) / Mutate Resources* -# Mutate Configurations +# Mutate Resources The ```mutate``` rule can be used to add, replace, or delete elements in matching resources. A mutate rule can be written as a JSON Patch or as an overlay. diff --git a/documentation/writing-policies-validate.md b/documentation/writing-policies-validate.md index 4a98b21d29..b8f09d5128 100644 --- a/documentation/writing-policies-validate.md +++ b/documentation/writing-policies-validate.md @@ -1,7 +1,7 @@ -*[documentation](/README.md#documentation) / [Writing Policies](/documentation/writing-policies.md) / Validate* +*[documentation](/README.md#documentation) / [Writing Policies](/documentation/writing-policies.md) / Validate Resources* -# Validate Configurations +# Validate Resources A validation rule is expressed as an overlay pattern that expresses the desired configuration. Resource configurations must match fields and expressions defined in the pattern to pass the validation rule. The following rules are followed when processing the overlay pattern: @@ -191,4 +191,4 @@ Additional examples are available in [samples](/samples/README.md) The `validationFailureAction` attribute controls processing behaviors when the resource is not compliant with the policy. If the value is set to `enforce` resource creation or updates are blocked when the resource does not comply, and when the value is set to `audit` a policy violation is reported but the resource creation or update is allowed. --- -*Read Next >> [Generate](/documentation/writing-policies-mutate.md)* +*Read Next >> [Generate](/documentation/writing-policies-generate.md)* From 5f1db2738efa27e84d2646a1fa9172427eb5ffe7 Mon Sep 17 00:00:00 2001 From: Jim Bugwadia Date: Thu, 6 Feb 2020 00:13:31 -0800 Subject: [PATCH 09/14] fix order --- README.md | 2 +- documentation/writing-policies-mutate.md | 2 +- documentation/writing-policies-validate.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 55a2381860..c3ef8c234f 100644 --- a/README.md +++ b/README.md @@ -123,8 +123,8 @@ Refer to a list of curated of ***[sample policies](/samples/README.md)*** that c * [Getting Started](documentation/installation.md) * [Writing Policies](documentation/writing-policies.md) - * [Mutate Resources](documentation/writing-policies-mutate.md) * [Validate Resources](documentation/writing-policies-validate.md) + * [Mutate Resources](documentation/writing-policies-mutate.md) * [Generate Resources](documentation/writing-policies-generate.md) * [Variable Substitution](documentation/writing-policies-variables.md) * [Preconditions](documentation/writing-policies-preconditions.md) diff --git a/documentation/writing-policies-mutate.md b/documentation/writing-policies-mutate.md index b9a91cb81b..74fee2946d 100644 --- a/documentation/writing-policies-mutate.md +++ b/documentation/writing-policies-mutate.md @@ -213,4 +213,4 @@ The anchor processing behavior for mutate conditions is as follows: Additional details on mutation overlay behaviors are available on the wiki: [Mutation Overlay](https://github.com/nirmata/kyverno/wiki/Mutation-Overlay) --- -*Read Next >> [Generate](/documentation/writing-policies-generate.md)* +*Read Next >> [Generate Resources](/documentation/writing-policies-generate.md)* diff --git a/documentation/writing-policies-validate.md b/documentation/writing-policies-validate.md index b8f09d5128..339bc0292f 100644 --- a/documentation/writing-policies-validate.md +++ b/documentation/writing-policies-validate.md @@ -191,4 +191,4 @@ Additional examples are available in [samples](/samples/README.md) The `validationFailureAction` attribute controls processing behaviors when the resource is not compliant with the policy. If the value is set to `enforce` resource creation or updates are blocked when the resource does not comply, and when the value is set to `audit` a policy violation is reported but the resource creation or update is allowed. --- -*Read Next >> [Generate](/documentation/writing-policies-generate.md)* +*Read Next >> [Mutate Resources](/documentation/writing-policies-mutate.md)* From 9615c60fba89cf2c47b87c85e93153d9fcd36d28 Mon Sep 17 00:00:00 2001 From: Jim Bugwadia Date: Thu, 6 Feb 2020 00:16:04 -0800 Subject: [PATCH 10/14] fix case and text --- documentation/writing-policies-autogen.md | 2 +- documentation/writing-policies.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/writing-policies-autogen.md b/documentation/writing-policies-autogen.md index baa9a2c314..97a6c922c6 100644 --- a/documentation/writing-policies-autogen.md +++ b/documentation/writing-policies-autogen.md @@ -1,6 +1,6 @@ *[documentation](/README.md#documentation) / [Writing Policies](/documentation/writing-policies.md) / Auto-Generation for Pod Controllers* -# Auto generating rules for pod controllers +# Auto Generating Rules for Pod Controllers Writing policies on pods helps address all pod creation flows. However, when pod cotrollers are used pod level policies result in errors not being reported when the pod controller object is created. diff --git a/documentation/writing-policies.md b/documentation/writing-policies.md index 3bb02ae2e3..e706598a72 100644 --- a/documentation/writing-policies.md +++ b/documentation/writing-policies.md @@ -63,4 +63,4 @@ Each rule can validate, mutate, or generate configurations of matching resources --- -*Read Next >> [Validate](/documentation/writing-policies-validate.md)* \ No newline at end of file +*Read Next >> [Validate Resources](/documentation/writing-policies-validate.md)* \ No newline at end of file From f188512ca0b5a40baa452df8c918043bcc6a39be Mon Sep 17 00:00:00 2001 From: Jim Bugwadia Date: Thu, 6 Feb 2020 00:38:05 -0800 Subject: [PATCH 11/14] add service accounts --- documentation/writing-policies.md | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/writing-policies.md b/documentation/writing-policies.md index e706598a72..1d017a4482 100644 --- a/documentation/writing-policies.md +++ b/documentation/writing-policies.md @@ -11,6 +11,7 @@ Each Kyverno policy contains one or more rules. Each rule has a `match` clause, The match / exclude clauses have the same structure, and can contain the following elements: * resources: select resources by name, namespaces, kinds, and label selectors. * subjects: select users and user groups +* serviceAccounts: select service accounts * roles: select namespaced roles * clusterroles: select cluster wide roles From 836cf11afd6394f86e6d1a693139f4b50b4fde40 Mon Sep 17 00:00:00 2001 From: Jim Bugwadia Date: Thu, 6 Feb 2020 00:42:01 -0800 Subject: [PATCH 12/14] add service accounts to subjects --- documentation/writing-policies.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/documentation/writing-policies.md b/documentation/writing-policies.md index 1d017a4482..cfede6f81e 100644 --- a/documentation/writing-policies.md +++ b/documentation/writing-policies.md @@ -10,8 +10,7 @@ Each Kyverno policy contains one or more rules. Each rule has a `match` clause, The match / exclude clauses have the same structure, and can contain the following elements: * resources: select resources by name, namespaces, kinds, and label selectors. -* subjects: select users and user groups -* serviceAccounts: select service accounts +* subjects: select users, user groups, and service accounts * roles: select namespaced roles * clusterroles: select cluster wide roles From d4bbae6fe8f67f9f2791ff8dd2132b57e6db8542 Mon Sep 17 00:00:00 2001 From: Jim Bugwadia Date: Thu, 6 Feb 2020 22:51:16 -0800 Subject: [PATCH 13/14] update examples and log text --- README.md | 21 ++++++++---------- documentation/writing-policies-generate.md | 25 ++++++++-------------- pkg/policyviolation/namespacedpv.go | 3 ++- 3 files changed, 20 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index c3ef8c234f..00264ce010 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,8 @@ kind: ClusterPolicy metadata: name: check-cpu-memory spec: + # `enforce` blocks request. `audit` reports violations + validationFailureAction: enforce rules: - name: check-pod-resources match: @@ -71,17 +73,15 @@ spec: match: resources: kinds: - - Deployment + - Pod mutate: overlay: spec: - template: - spec: - containers: - # match images which end with :latest - - (image): "*:latest" - # set the imagePullPolicy to "Always" - imagePullPolicy: "Always" + containers: + # match images which end with :latest + - (image): "*:latest" + # set the imagePullPolicy to "Always" + imagePullPolicy: "Always" ```` ### 3. Generating resources @@ -100,13 +100,10 @@ spec: resources: kinds: - Namespace - selector: - matchExpressions: - - {key: kafka, operator: Exists} generate: kind: ConfigMap name: zk-kafka-address - # create the resource in the new namespace + # generate the resource in the new namespace namespace: "{{request.object.metadata.name}}" data: kind: ConfigMap diff --git a/documentation/writing-policies-generate.md b/documentation/writing-policies-generate.md index 6780e96840..b8882590b1 100644 --- a/documentation/writing-policies-generate.md +++ b/documentation/writing-policies-generate.md @@ -5,8 +5,7 @@ ```generate``` is used to create additional resources when a resource is created. This is useful to create supporting resources, such as role bindings for a new namespace. ## Example 1 -- rule -Creates a ConfigMap with name `default-config` for all + ````yaml apiVersion: kyverno.io/v1 kind: ClusterPolicy @@ -19,28 +18,22 @@ spec: resources: kinds: - Namespace - selector: - matchLabels: - LabelForSelector : "namespace2" generate: kind: ConfigMap # Kind of resource name: default-config # Name of the new Resource - namespace: "{{request.object.metadata.name}}" # Create in the namespace that triggers this rule + namespace: "{{request.object.metadata.name}}" # namespace that triggers this rule clone: namespace: default name: config-template - - name: "Generate Secret" + - name: "Generate Secret (insecure)" match: resources: kinds: - Namespace - selector: - matchLabels: - LabelForSelector : "namespace2" generate: kind: Secret name: mongo-creds - namespace: "{{request.object.metadata.name}}" # Create in the namespace that triggers this rule + namespace: "{{request.object.metadata.name}}" # namespace that triggers this rule data: data: DB_USER: YWJyYWthZGFicmE= @@ -50,9 +43,9 @@ spec: purpose: mongo ```` -In this example, when this policy is applied, any new namespace that satisfies the label selector will receive 2 new resources after its creation: - * ConfigMap copied from default/config-template. - * Secret with values DB_USER and DB_PASSWORD, and label ```purpose: mongo```. +In this example new namespaces will receive 2 new resources after its creation: + * A ConfigMap cloned from default/config-template. + * A Secret with values DB_USER and DB_PASSWORD, and label ```purpose: mongo```. ## Example 2 @@ -72,7 +65,7 @@ spec: generate: kind: NetworkPolicy name: deny-all-traffic - namespace: "{{request.object.metadata.name}}" # Create in the namespace that triggers this rule + namespace: "{{request.object.metadata.name}}" # namespace that triggers this rule data: spec: podSelector: @@ -84,7 +77,7 @@ spec: policyname: "default" ```` -In this example, when the policy is applied, any new namespace will receive a NetworkPolicy based on the specified template that by default denies all inbound and outbound traffic. +In this example new namespaces will receive a NetworkPolicy that default denies all inbound and outbound traffic. --- diff --git a/pkg/policyviolation/namespacedpv.go b/pkg/policyviolation/namespacedpv.go index ad927b3ea4..1967a0ba3f 100644 --- a/pkg/policyviolation/namespacedpv.go +++ b/pkg/policyviolation/namespacedpv.go @@ -109,8 +109,9 @@ func (nspv *namespacedPV) updatePV(newPv, oldPv *kyverno.PolicyViolation) error // update resource _, err = nspv.kyvernoInterface.PolicyViolations(newPv.GetNamespace()).Update(newPv) if err != nil { - return fmt.Errorf("failed to update namespaced polciy violation: %v", err) + return fmt.Errorf("failed to update namespaced policy violation: %v", err) } + glog.Infof("namespaced policy violation updated for resource %v", newPv.Spec.ResourceSpec) return nil } From 53637a6ccfb9e2fba0634bd131d409a516ed8d35 Mon Sep 17 00:00:00 2001 From: Jim Bugwadia Date: Thu, 6 Feb 2020 23:20:31 -0800 Subject: [PATCH 14/14] fix text --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 00264ce010..bbc6721e1d 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ kind: ClusterPolicy metadata: name: check-cpu-memory spec: - # `enforce` blocks request. `audit` reports violations + # `enforce` blocks the request. `audit` reports violations validationFailureAction: enforce rules: - name: check-pod-resources