From e87c72291f10801b11b3b8947f6759e591006707 Mon Sep 17 00:00:00 2001 From: Shuting Zhao Date: Wed, 14 Aug 2019 11:51:01 -0700 Subject: [PATCH] - Patch resource between every rule application - move mutation & validation to mutate webhook --- pkg/engine/engine.go | 2 +- pkg/engine/mutation.go | 23 +++++++---- pkg/webhooks/mutation.go | 61 ++++++++++++++--------------- pkg/webhooks/registration.go | 13 ++++--- pkg/webhooks/server.go | 28 ++++++++++++-- pkg/webhooks/utils.go | 2 + pkg/webhooks/validation.go | 30 +++++++------- pkg/webhooks/webhookManager.go | 71 +++------------------------------- 8 files changed, 98 insertions(+), 132 deletions(-) diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 5698c51048..49332c7646 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -61,7 +61,7 @@ func applyPolicy(client *client.Client, policy *types.Policy, res resourceInfo) } func mutation(p *types.Policy, rawResource []byte, gvk *metav1.GroupVersionKind) ([]*info.RuleInfo, error) { - patches, ruleInfos := Mutate(*p, rawResource, *gvk) + patches, _, ruleInfos := Mutate(*p, rawResource, *gvk) if len(ruleInfos) == 0 { // no rules were processed return nil, nil diff --git a/pkg/engine/mutation.go b/pkg/engine/mutation.go index f1d551e1e6..a518e9fb4f 100644 --- a/pkg/engine/mutation.go +++ b/pkg/engine/mutation.go @@ -8,8 +8,10 @@ import ( ) // Mutate performs mutation. Overlay first and then mutation patches -func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersionKind) ([][]byte, []*info.RuleInfo) { - var allPatches [][]byte +func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersionKind) ([][]byte, []byte, []*info.RuleInfo) { + var allPatches, rulePatches [][]byte + var err error + var errs []error patchedDocument := rawResource ris := []*info.RuleInfo{} @@ -26,9 +28,9 @@ func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersio } // Process Overlay if rule.Mutation.Overlay != nil { - overlayPatches, err := ProcessOverlay(rule, rawResource, gvk) + rulePatches, err = ProcessOverlay(rule, patchedDocument, gvk) if err == nil { - if len(overlayPatches) == 0 { + if len(rulePatches) == 0 { // if array elements dont match then we skip(nil patch, no error) // or if acnohor is defined and doenst match // policy is not applicable @@ -36,10 +38,10 @@ func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersio } ri.Addf("Rule %s: Overlay succesfully applied.", rule.Name) // merge the json patches - patch := JoinPatches(overlayPatches) + patch := JoinPatches(rulePatches) // strip slashes from string ri.Changes = string(patch) - allPatches = append(allPatches, overlayPatches...) + allPatches = append(allPatches, rulePatches...) } else { ri.Fail() ri.Addf("overlay application has failed, err %v.", err) @@ -48,7 +50,7 @@ func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersio // Process Patches if len(rule.Mutation.Patches) != 0 { - rulePatches, errs := ProcessPatches(rule, patchedDocument) + rulePatches, errs = ProcessPatches(rule, patchedDocument) if len(errs) > 0 { ri.Fail() for _, err := range errs { @@ -59,8 +61,13 @@ func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersio allPatches = append(allPatches, rulePatches...) } } + + patchedDocument, err = ApplyPatches(rawResource, rulePatches) + if err != nil { + glog.Errorf("Failed to apply patches on ruleName=%s, err%v\n:", rule.Name, err) + } ris = append(ris, ri) } - return allPatches, ris + return allPatches, patchedDocument, ris } diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go index 82c5f01aac..41f78e177b 100644 --- a/pkg/webhooks/mutation.go +++ b/pkg/webhooks/mutation.go @@ -5,12 +5,19 @@ import ( engine "github.com/nirmata/kyverno/pkg/engine" "github.com/nirmata/kyverno/pkg/info" v1beta1 "k8s.io/api/admission/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ) // HandleMutation handles mutating webhook admission request -func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { +func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) (bool, [][]byte, []byte) { + var allPatches, policyPatches [][]byte + policyInfos := []*info.PolicyInfo{} + var ruleInfos []*info.RuleInfo + patchedDocument := request.Object.Raw + + if request.Operation == v1beta1.Delete { + return true, nil, patchedDocument + } glog.V(4).Infof("Receive request in mutating webhook: Kind=%s, Namespace=%s Name=%s UID=%s patchOperation=%s", request.Kind.Kind, request.Namespace, request.Name, request.UID, request.Operation) @@ -18,12 +25,11 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be policies, err := ws.policyLister.List(labels.NewSelector()) if err != nil { // Unable to connect to policy Lister to access policies - glog.Error("Unable to connect to policy controller to access policies. Mutation Rules are NOT being applied") + glog.Errorln("Unable to connect to policy controller to access policies. Mutation Rules are NOT being applied") glog.Warning(err) - return &v1beta1.AdmissionResponse{ - Allowed: true, - } + return true, nil, patchedDocument } + rname := engine.ParseNameFromObject(request.Object.Raw) rns := engine.ParseNamespaceFromObject(request.Object.Raw) rkind := request.Kind.Kind @@ -31,19 +37,17 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be glog.Errorf("failed to parse KIND from request: Namespace=%s Name=%s UID=%s patchOperation=%s\n", request.Namespace, request.Name, request.UID, request.Operation) } - var allPatches [][]byte - policyInfos := []*info.PolicyInfo{} for _, policy := range policies { + // check if policy has a rule for the admission request kind if !StringInSlice(request.Kind.Kind, getApplicableKindsForPolicy(policy)) { continue } //TODO: HACK Check if an update of annotations - if checkIfOnlyAnnotationsUpdate(request) { - return &v1beta1.AdmissionResponse{ - Allowed: true, - } - } + // if checkIfOnlyAnnotationsUpdate(request) { + // return true + // } + policyInfo := info.NewPolicyInfo(policy.Name, rkind, rname, @@ -55,7 +59,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be glog.Infof("Applying policy %s with %d rules\n", policy.ObjectMeta.Name, len(policy.Spec.Rules)) - policyPatches, ruleInfos := engine.Mutate(*policy, request.Object.Raw, request.Kind) + policyPatches, patchedDocument, ruleInfos = engine.Mutate(*policy, patchedDocument, request.Kind) policyInfo.AddRuleInfos(ruleInfos) @@ -71,34 +75,27 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be glog.Info(err) } allPatches = append(allPatches, policyPatches...) - glog.Infof("Mutation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) + glog.Infof("Mutation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rns, rname) } policyInfos = append(policyInfos, policyInfo) - annPatch := addAnnotationsToResource(request.Object.Raw, policyInfo, info.Mutation) - if annPatch != nil { - // add annotations - ws.annotationsController.Add(rkind, rns, rname, annPatch) - } + // annPatch := addAnnotationsToResource(patchedDocument, policyInfo, info.Mutation) + // if annPatch != nil { + // // add annotations + // ws.annotationsController.Add(rkind, rns, rname, annPatch) + // } } if len(allPatches) > 0 { eventsInfo, _ := newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update), info.Mutation) ws.eventController.Add(eventsInfo...) } + ok, msg := isAdmSuccesful(policyInfos) if ok { - patchType := v1beta1.PatchTypeJSONPatch - return &v1beta1.AdmissionResponse{ - Allowed: true, - Patch: engine.JoinPatches(allPatches), - PatchType: &patchType, - } - } - return &v1beta1.AdmissionResponse{ - Allowed: false, - Result: &metav1.Status{ - Message: msg, - }, + return true, allPatches, patchedDocument } + + glog.Errorf("Failed to mutate the resource: %s\n", msg) + return false, nil, patchedDocument } diff --git a/pkg/webhooks/registration.go b/pkg/webhooks/registration.go index e0248fa86f..739c64392e 100644 --- a/pkg/webhooks/registration.go +++ b/pkg/webhooks/registration.go @@ -11,6 +11,7 @@ import ( client "github.com/nirmata/kyverno/pkg/dclient" "github.com/tevino/abool" admregapi "k8s.io/api/admissionregistration/v1beta1" + errorsapi "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" admregclient "k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1" rest "k8s.io/client-go/rest" @@ -112,12 +113,12 @@ func (wrc *WebhookRegistrationClient) DeregisterAll() { if wrc.serverIP != "" { err := wrc.registrationClient.ValidatingWebhookConfigurations().Delete(config.PolicyValidatingWebhookConfigurationDebug, &v1.DeleteOptions{}) - if err != nil { + if err != nil && !errorsapi.IsNotFound(err) { glog.Error(err) } } err := wrc.registrationClient.ValidatingWebhookConfigurations().Delete(config.PolicyValidatingWebhookConfigurationName, &v1.DeleteOptions{}) - if err != nil { + if err != nil && !errorsapi.IsNotFound(err) { glog.Error(err) } } @@ -130,7 +131,7 @@ func (wrc *WebhookRegistrationClient) deregister() { func (wrc *WebhookRegistrationClient) deregisterMutatingWebhook() { if wrc.serverIP != "" { err := wrc.registrationClient.MutatingWebhookConfigurations().Delete(config.MutatingWebhookConfigurationDebug, &v1.DeleteOptions{}) - if err != nil { + if err != nil && !errorsapi.IsNotFound(err) { glog.Error(err) } else { wrc.MutationRegistered.UnSet() @@ -139,7 +140,7 @@ func (wrc *WebhookRegistrationClient) deregisterMutatingWebhook() { } err := wrc.registrationClient.MutatingWebhookConfigurations().Delete(config.MutatingWebhookConfigurationName, &v1.DeleteOptions{}) - if err != nil { + if err != nil && !errorsapi.IsNotFound(err) { glog.Error(err) } else { wrc.MutationRegistered.UnSet() @@ -149,7 +150,7 @@ func (wrc *WebhookRegistrationClient) deregisterMutatingWebhook() { func (wrc *WebhookRegistrationClient) deregisterValidatingWebhook() { if wrc.serverIP != "" { err := wrc.registrationClient.ValidatingWebhookConfigurations().Delete(config.ValidatingWebhookConfigurationDebug, &v1.DeleteOptions{}) - if err != nil { + if err != nil && !errorsapi.IsNotFound(err) { glog.Error(err) } wrc.ValidationRegistered.UnSet() @@ -157,7 +158,7 @@ func (wrc *WebhookRegistrationClient) deregisterValidatingWebhook() { } err := wrc.registrationClient.ValidatingWebhookConfigurations().Delete(config.ValidatingWebhookConfigurationName, &v1.DeleteOptions{}) - if err != nil { + if err != nil && !errorsapi.IsNotFound(err) { glog.Error(err) } wrc.ValidationRegistered.UnSet() diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index a0e309234b..b166054355 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -10,6 +10,8 @@ import ( "net/http" "time" + "github.com/nirmata/kyverno/pkg/engine" + "github.com/golang/glog" "github.com/nirmata/kyverno/pkg/annotations" "github.com/nirmata/kyverno/pkg/client/listers/policy/v1alpha1" @@ -114,13 +116,10 @@ func (ws *WebhookServer) serve(w http.ResponseWriter, r *http.Request) { // Resource UPDATE switch r.URL.Path { case config.MutatingWebhookServicePath: - admissionReview.Response = ws.HandleMutation(admissionReview.Request) - case config.ValidatingWebhookServicePath: - admissionReview.Response = ws.HandleValidation(admissionReview.Request) + admissionReview.Response = ws.HandleAdmissionRequest(admissionReview.Request) case config.PolicyValidatingWebhookServicePath: admissionReview.Response = ws.HandlePolicyValidation(admissionReview.Request) } - } } @@ -138,6 +137,27 @@ func (ws *WebhookServer) serve(w http.ResponseWriter, r *http.Request) { } } +func (ws *WebhookServer) HandleAdmissionRequest(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { + var response *v1beta1.AdmissionResponse + + allowed, allPatches, patchedDocument := ws.HandleMutation(request) + if !allowed { + // TODO: add failure message to response + return &v1beta1.AdmissionResponse{ + Allowed: false, + } + } + + response = ws.HandleValidation(request, patchedDocument) + if response.Allowed && len(allPatches) > 0 { + patchType := v1beta1.PatchTypeJSONPatch + response.Patch = engine.JoinPatches(allPatches) + response.PatchType = &patchType + } + + return response +} + // RunAsync TLS server in separate thread and returns control immediately func (ws *WebhookServer) RunAsync() { go func(ws *WebhookServer) { diff --git a/pkg/webhooks/utils.go b/pkg/webhooks/utils.go index 83ca8944d7..1b055ab8f9 100644 --- a/pkg/webhooks/utils.go +++ b/pkg/webhooks/utils.go @@ -109,9 +109,11 @@ const ( func toBlock(pis []*info.PolicyInfo) bool { for _, pi := range pis { if pi.ValidationFailureAction != ReportViolation { + glog.V(3).Infoln("ValidationFailureAction set to enforce, blocking resource ceation") return true } } + glog.V(3).Infoln("ValidationFailureAction set to audit, allowing resource creation, reporting with violation") return false } diff --git a/pkg/webhooks/validation.go b/pkg/webhooks/validation.go index b444c4c11c..abeca7f792 100644 --- a/pkg/webhooks/validation.go +++ b/pkg/webhooks/validation.go @@ -11,7 +11,7 @@ import ( // HandleValidation handles validating webhook admission request // If there are no errors in validating rule we apply generation rules -func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { +func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest, rawResource []byte) *v1beta1.AdmissionResponse { glog.V(4).Infof("Receive request in validating webhook: Kind=%s, Namespace=%s Name=%s UID=%s patchOperation=%s", request.Kind.Kind, request.Namespace, request.Name, request.UID, request.Operation) @@ -27,8 +27,8 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 } } - rname := engine.ParseNameFromObject(request.Object.Raw) - rns := engine.ParseNamespaceFromObject(request.Object.Raw) + rname := engine.ParseNameFromObject(rawResource) + rns := engine.ParseNamespaceFromObject(rawResource) rkind := request.Kind.Kind if rkind == "" { glog.Errorf("failed to parse KIND from request: Namespace=%s Name=%s UID=%s patchOperation=%s\n", request.Namespace, request.Name, request.UID, request.Operation) @@ -40,12 +40,12 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 continue } //TODO: HACK Check if an update of annotations - if checkIfOnlyAnnotationsUpdate(request) { - // allow the update of resource to add annotations - return &v1beta1.AdmissionResponse{ - Allowed: true, - } - } + // if checkIfOnlyAnnotationsUpdate(request) { + // // allow the update of resource to add annotations + // return &v1beta1.AdmissionResponse{ + // Allowed: true, + // } + // } policyInfo := info.NewPolicyInfo(policy.Name, rkind, @@ -57,7 +57,7 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 request.Kind.Kind, rns, rname, request.UID, request.Operation) glog.Infof("Validating resource %s/%s/%s with policy %s with %d rules", rkind, rns, rname, policy.ObjectMeta.Name, len(policy.Spec.Rules)) - ruleInfos, err := engine.Validate(*policy, request.Object.Raw, request.Kind) + ruleInfos, err := engine.Validate(*policy, rawResource, request.Kind) if err != nil { // This is not policy error // but if unable to parse request raw resource @@ -84,11 +84,11 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 } } policyInfos = append(policyInfos, policyInfo) - // annotations - annPatch := addAnnotationsToResource(request.Object.Raw, policyInfo, info.Validation) - if annPatch != nil { - ws.annotationsController.Add(rkind, rns, rname, annPatch) - } + // // annotations + // annPatch := addAnnotationsToResource(request.Object.Raw, policyInfo, info.Validation) + // if annPatch != nil { + // ws.annotationsController.Add(rkind, rns, rname, annPatch) + // } } if len(policyInfos) > 0 && len(policyInfos[0].Rules) != 0 { diff --git a/pkg/webhooks/webhookManager.go b/pkg/webhooks/webhookManager.go index 118d31a4f8..5f8ec0a760 100644 --- a/pkg/webhooks/webhookManager.go +++ b/pkg/webhooks/webhookManager.go @@ -33,79 +33,18 @@ func (ws *WebhookServer) registerWebhookConfigurations(policy v1alpha1.Policy) e } glog.Infof("Mutating webhook registered") } - - if rule.Validation != nil && !ws.webhookRegistrationClient.ValidationRegistered.IsSet() { - if err := ws.webhookRegistrationClient.RegisterValidatingWebhook(); err != nil { - return err - } - glog.Infof("Validating webhook registered") - } } return nil } func (ws *WebhookServer) deregisterWebhookConfigurations(policy v1alpha1.Policy) error { - glog.V(3).Infof("Retreiving policy type for %s\n", policy.Name) + policies, _ := ws.policyLister.List(labels.NewSelector()) - pt := GetPolicyType([]*v1alpha1.Policy{&policy}, "") - - glog.V(3).Infof("Policy to be deleted type==%v\n", pt) - - existPolicyType := ws.getExistingPolicyType(policy.Name) - glog.V(3).Infof("Found existing policy type==%v\n", existPolicyType) - - switch existPolicyType { - case none: - ws.webhookRegistrationClient.deregister() - glog.Infoln("All webhook deregistered") - case mutate: - if pt != mutate { - ws.webhookRegistrationClient.deregisterValidatingWebhook() - glog.Infoln("Validating webhook deregistered") - } - case validate: - if pt != validate { - ws.webhookRegistrationClient.deregisterMutatingWebhook() - glog.Infoln("Mutating webhook deregistered") - } - case all: - return nil + // deregister webhook if no policy found in cluster + if len(policies) == 1 { + ws.webhookRegistrationClient.deregisterMutatingWebhook() + glog.Infoln("Mutating webhook deregistered") } return nil } - -func (ws *WebhookServer) getExistingPolicyType(policyName string) policyType { - - policies, err := ws.policyLister.List(labels.NewSelector()) - if err != nil { - glog.Errorf("Failed to get policy list") - } - - return GetPolicyType(policies, policyName) -} - -// GetPolicyType get the type of policies -// excludes is the policy name to be skipped -func GetPolicyType(policyList []*v1alpha1.Policy, excludes string) policyType { - ptype := none - - for _, p := range policyList { - if p.Name == excludes { - glog.Infof("Skipping policy type check on %s\n", excludes) - continue - } - - for _, rule := range p.Spec.Rules { - if rule.Mutation != nil { - ptype = ptype | mutate - } - - if rule.Validation != nil { - ptype = ptype | validate - } - } - } - - return ptype -}