From 29a89d20ada0c176171b552ee7a345d0bed0d740 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Mon, 15 Jul 2019 11:29:58 -0700 Subject: [PATCH 01/24] violation cleanup for existing resources --- pkg/apis/policy/v1alpha1/types.go | 13 +- pkg/apis/policy/v1alpha1/utils.go | 21 +++ .../policy/v1alpha1/zz_generated.deepcopy.go | 16 ++- pkg/controller/controller.go | 4 +- pkg/engine/engine.go | 2 +- pkg/violation/builder.go | 129 ++++++++++++++---- pkg/violation/util.go | 10 ++ pkg/webhooks/server.go | 21 ++- 8 files changed, 172 insertions(+), 44 deletions(-) diff --git a/pkg/apis/policy/v1alpha1/types.go b/pkg/apis/policy/v1alpha1/types.go index 889d1bf775..91f1f92a4e 100644 --- a/pkg/apis/policy/v1alpha1/types.go +++ b/pkg/apis/policy/v1alpha1/types.go @@ -77,16 +77,17 @@ type CloneFrom struct { // Status contains violations for existing resources type Status struct { - Violations []Violation `json:"violations,omitempty"` + // Violations map[kind/namespace/resource]Violation + Violations map[string]Violation `json:"violations,omitempty"` } // Violation for the policy type Violation struct { - Kind string `json:"kind,omitempty"` - Name string `json:"name,omitempty"` - Namespace string `json:"namespace,omitempty"` - Reason string `json:"reason,omitempty"` - Message string `json:"message,omitempty"` + Kind string `json:"kind,omitempty"` + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` + Rules []string `json:"rules"` + Reason string `json:"reason,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/policy/v1alpha1/utils.go b/pkg/apis/policy/v1alpha1/utils.go index 59adc88a5d..17c067c5fb 100644 --- a/pkg/apis/policy/v1alpha1/utils.go +++ b/pkg/apis/policy/v1alpha1/utils.go @@ -104,3 +104,24 @@ func (in *Generation) DeepCopyInto(out *Generation) { *out = *in } } + +//IsEqual Check if violatiosn are equal +func (v *Violation) IsEqual(nv Violation) bool { + // We do not need to compare resource info as it will be same + // Reason + if v.Reason != nv.Reason { + return false + } + // Rule + if len(v.Rules) != len(nv.Rules) { + return false + } + // assumes the rules will be in order, as the rule are proceeed in order + // if the rule order changes, it means the policy has changed.. as it will afffect the order in which mutation rules are applied + for i, r := range v.Rules { + if r != nv.Rules[i] { + return false + } + } + return true +} diff --git a/pkg/apis/policy/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/policy/v1alpha1/zz_generated.deepcopy.go index 903ae1f40c..24be64c1b6 100644 --- a/pkg/apis/policy/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/policy/v1alpha1/zz_generated.deepcopy.go @@ -135,6 +135,11 @@ func (in *ResourceDescription) DeepCopyInto(out *ResourceDescription) { *out = new(string) **out = **in } + if in.Namespace != nil { + in, out := &in.Namespace, &out.Namespace + *out = new(string) + **out = **in + } if in.Selector != nil { in, out := &in.Selector, &out.Selector *out = new(v1.LabelSelector) @@ -210,8 +215,10 @@ func (in *Status) DeepCopyInto(out *Status) { *out = *in if in.Violations != nil { in, out := &in.Violations, &out.Violations - *out = make([]Violation, len(*in)) - copy(*out, *in) + *out = make(map[string]Violation, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } } return } @@ -239,6 +246,11 @@ func (in *Validation) DeepCopy() *Validation { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Violation) DeepCopyInto(out *Violation) { *out = *in + if in.Rules != nil { + in, out := &in.Rules, &out.Rules + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 01347ee09a..d8730adaf8 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -225,7 +225,9 @@ func createEventsAndViolations(eventController event.Generator, policyInfos []*i e := event.NewEvent("Policy", "", policyInfo.Name, event.PolicyViolation, event.FResourcePolcy, policyInfo.RNamespace+"/"+policyInfo.RName, strings.Join(fruleNames, ";")) events = append(events, e) // Violation - v := violation.NewViolationFromEvent(e, policyInfo.Name, policyInfo.RKind, policyInfo.RName, policyInfo.RNamespace) + // TODO: Violation is currently create at policy, level not resource level + // As we create violation, we check if the + v := violation.BuldNewViolation(policyInfo.Name, policyInfo.RKind, policyInfo.RNamespace, policyInfo.RName, event.PolicyViolation.String(), fruleNames) violations = append(violations, v) } // else { diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index d61545fb8f..4cc94d80e5 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -34,7 +34,7 @@ func ProcessExisting(client *client.Client, policy *types.Policy) []*info.Policy if rule.ResourceDescription.Namespace != nil { namespace = *rule.ResourceDescription.Namespace } - list, err := client.ListResource(gvr.Resource, namespace, rule.ResourceDescription.Selector) + list, err := client.ListResource(k, namespace, rule.ResourceDescription.Selector) if err != nil { glog.Errorf("unable to list resource for %s with label selector %s", gvr.Resource, rule.Selector.String()) glog.Errorf("unable to apply policy %s rule %s. err: %s", policy.Name, rule.Name, err) diff --git a/pkg/violation/builder.go b/pkg/violation/builder.go index 7fe9645274..ff48c55e8f 100644 --- a/pkg/violation/builder.go +++ b/pkg/violation/builder.go @@ -51,7 +51,7 @@ func (b *builder) Add(infos ...*Info) error { } func (b *builder) processViolation(info *Info) error { - currentViolations := []interface{}{} + currVs := map[string]interface{}{} statusMap := map[string]interface{}{} var ok bool //TODO: hack get from client @@ -71,37 +71,71 @@ func (b *builder) processViolation(info *Info) error { if !ok { glog.Info("violation not present") } + // Violations map[string][]Violations glog.Info(reflect.TypeOf(violations)) - if currentViolations, ok = violations.([]interface{}); !ok { + if currVs, ok = violations.(map[string]interface{}); !ok { return errors.New("Unable to parse violations") } } - newViolation := info.Violation - for _, violation := range currentViolations { - glog.Info(reflect.TypeOf(violation)) - if v, ok := violation.(map[string]interface{}); ok { - if name, ok := v["name"].(string); ok { - if namespace, ok := v["namespace"].(string); ok { - ok, err := b.isActive(info.Kind, name, namespace) - if err != nil { - glog.Error(err) - continue - } - if !ok { - //TODO remove the violation as it corresponds to resource that does not exist - glog.Info("removed violation") - } - } - } + // Info: + // Resource - Kind, Namespace, Name + // policy - Name + // violation, ok := currVs[info.getKey()] + // Key -> resource + // 1> Check if there were any previous violations for the given key + // 2> If No, create a new one + if !ok { + currVs[info.getKey()] = info.Violation + } else { + currV := currVs[info.getKey()] + glog.Info(reflect.TypeOf(currV)) + v, ok := currV.(map[string]interface{}) + if !ok { + glog.Info("type not matching") } + // get rules + rules, ok := v["rules"] + if !ok { + glog.Info("rules not found") + } + glog.Info(reflect.TypeOf(rules)) + rs, ok := rules.([]interface{}) + if !ok { + glog.Info("type not matching") + } + // check if rules are samre + if isRuleNamesEqual(rs, info.Violation.Rules) { + return nil + } + // else update the errors + currVs[info.getKey()] = info.Violation } - currentViolations = append(currentViolations, newViolation) - // update violations + // newViolation := info.Violation + // for _, violation := range currentViolations { + // glog.Info(reflect.TypeOf(violation)) + // if v, ok := violation.(map[string]interface{}); ok { + // if name, ok := v["name"].(string); ok { + // if namespace, ok := v["namespace"].(string); ok { + // ok, err := b.isActive(info.Kind, name, namespace) + // if err != nil { + // glog.Error(err) + // continue + // } + // if !ok { + // //TODO remove the violation as it corresponds to resource that does not exist + // glog.Info("removed violation") + // } + // } + // } + // } + // } + // currentViolations = append(currentViolations, newViolation) + // // update violations // set the updated status - statusMap["violations"] = currentViolations + statusMap["violations"] = currVs unstr["status"] = statusMap p1.SetUnstructuredContent(unstr) - _, err = b.client.UpdateStatusResource("policies", "", p1, false) + _, err = b.client.UpdateStatusResource("Policy", "", p1, false) if err != nil { return err } @@ -126,21 +160,58 @@ func NewViolation(reason event.Reason, policyName, kind, rname, rnamespace, msg Name: rname, Namespace: rnamespace, Reason: reason.String(), - Message: msg, }, } } -//NewViolationFromEvent returns violation info from event -func NewViolationFromEvent(e *event.Info, pName, rKind, rName, rnamespace string) *Info { +// //NewViolationFromEvent returns violation info from event +// func NewViolationFromEvent(e *event.Info, pName, rKind, rName, rnamespace string) *Info { +// return &Info{ +// Policy: pName, +// Violation: types.Violation{ +// Kind: rKind, +// Name: rName, +// Namespace: rnamespace, +// Reason: e.Reason, +// Message: e.Message, +// }, +// } +// } +// Build a new Violation +func BuldNewViolation(pName string, rKind string, rNs string, rName string, reason string, rules []string) *Info { return &Info{ Policy: pName, Violation: types.Violation{ Kind: rKind, + Namespace: rNs, Name: rName, - Namespace: rnamespace, - Reason: e.Reason, - Message: e.Message, + Reason: reason, + Rules: rules, }, } } + +func isRuleNamesEqual(currRules []interface{}, newRules []string) bool { + if len(currRules) != len(newRules) { + return false + } + for i, r := range currRules { + name, ok := r.(string) + if !ok { + return false + } + if name != newRules[i] { + return false + } + } + return true +} + +//RemoveViolation will remove the violation for the resource if there was one +func RemoveViolation(policy *types.Policy, rKind string, rNs string, rName string) { + // Remove the pair from map + if policy.Status.Violations != nil { + glog.Infof("Cleaning up violalation for policy %s, resource %s/%s/%s", policy.Name, rKind, rNs, rName) + delete(policy.Status.Violations, BuildKey(rKind, rNs, rName)) + } +} diff --git a/pkg/violation/util.go b/pkg/violation/util.go index e83b9916ab..bd0b9a1c3a 100644 --- a/pkg/violation/util.go +++ b/pkg/violation/util.go @@ -16,3 +16,13 @@ type Info struct { Policy string policytype.Violation } + +func (i Info) getKey() string { + return i.Kind + "/" + i.Namespace + "/" + i.Name +} + +//BuildKey returns the key format +func BuildKey(rKind, rNs, rName string) string { + return rKind + "/" + rNs + "/" + rName +} + diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index b38e3f9796..29b9830629 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -22,6 +22,7 @@ import ( "github.com/nirmata/kyverno/pkg/sharedinformer" tlsutils "github.com/nirmata/kyverno/pkg/tls" "github.com/nirmata/kyverno/pkg/utils" + "github.com/nirmata/kyverno/pkg/violation" v1beta1 "k8s.io/api/admission/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -182,9 +183,14 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be for _, r := range ruleInfos { glog.Warning(r.Msgs) } - } else if len(policyPatches) > 0 { - allPatches = append(allPatches, policyPatches...) - glog.Infof("Mutation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) + } else { + fmt.Println("cleanup") + // CleanUp Violations if exists + violation.RemoveViolation(policy, request.Kind.Kind, rns, rname) + if len(policyPatches) > 0 { + allPatches = append(allPatches, policyPatches...) + glog.Infof("Mutation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) + } } policyInfos = append(policyInfos, policyInfo) } @@ -274,8 +280,13 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 for _, r := range ruleInfos { glog.Warning(r.Msgs) } - } else if len(ruleInfos) > 0 { - glog.Infof("Validation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) + } else { + // CleanUp Violations if exists + violation.RemoveViolation(policy, request.Kind.Kind, rns, rname) + + if len(ruleInfos) > 0 { + glog.Infof("Validation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) + } } policyInfos = append(policyInfos, policyInfo) } From a5817f5863d5082f0922fa293d1e68ecd2a1a9d6 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Mon, 15 Jul 2019 14:49:22 -0700 Subject: [PATCH 02/24] violation clean up --- main.go | 2 +- pkg/apis/policy/v1alpha1/types.go | 16 ++- .../policy/v1alpha1/zz_generated.deepcopy.go | 18 ++- pkg/controller/controller.go | 40 ++++-- pkg/controller/utils.go | 14 ++ pkg/violation/builder.go | 129 ++++++++++++++++-- pkg/webhooks/server.go | 35 +++-- 7 files changed, 207 insertions(+), 47 deletions(-) diff --git a/main.go b/main.go index b27405c47e..a228f43532 100644 --- a/main.go +++ b/main.go @@ -55,7 +55,7 @@ func main() { if err != nil { glog.Fatalf("Failed to initialize TLS key/certificate pair: %v\n", err) } - server, err := webhooks.NewWebhookServer(client, tlsPair, policyInformerFactory, eventController, filterK8Kinds) + server, err := webhooks.NewWebhookServer(client, tlsPair, policyInformerFactory, eventController, violationBuilder, filterK8Kinds) if err != nil { glog.Fatalf("Unable to create webhook server: %v\n", err) } diff --git a/pkg/apis/policy/v1alpha1/types.go b/pkg/apis/policy/v1alpha1/types.go index 91f1f92a4e..2d30a9e181 100644 --- a/pkg/apis/policy/v1alpha1/types.go +++ b/pkg/apis/policy/v1alpha1/types.go @@ -83,11 +83,17 @@ type Status struct { // Violation for the policy type Violation struct { - Kind string `json:"kind,omitempty"` - Name string `json:"name,omitempty"` - Namespace string `json:"namespace,omitempty"` - Rules []string `json:"rules"` - Reason string `json:"reason,omitempty"` + Kind string `json:"kind,omitempty"` + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` + Rules []FailedRule `json:"rules"` + Reason string `json:"reason,omitempty"` +} + +// FailedRule stored info and type of failed rules +type FailedRule struct { + Name string `json:"name"` + Type string `json:"type"` //Mutation, Validation, Genertaion } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/policy/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/policy/v1alpha1/zz_generated.deepcopy.go index 24be64c1b6..627535f830 100644 --- a/pkg/apis/policy/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/policy/v1alpha1/zz_generated.deepcopy.go @@ -41,6 +41,22 @@ func (in *CloneFrom) DeepCopy() *CloneFrom { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FailedRule) DeepCopyInto(out *FailedRule) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FailedRule. +func (in *FailedRule) DeepCopy() *FailedRule { + if in == nil { + return nil + } + out := new(FailedRule) + in.DeepCopyInto(out) + return out +} + // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Generation. func (in *Generation) DeepCopy() *Generation { if in == nil { @@ -248,7 +264,7 @@ func (in *Violation) DeepCopyInto(out *Violation) { *out = *in if in.Rules != nil { in, out := &in.Rules, &out.Rules - *out = make([]string, len(*in)) + *out = make([]FailedRule, len(*in)) copy(*out, *in) } return diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index d8730adaf8..524a144a1b 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -3,7 +3,6 @@ package controller import ( "fmt" "reflect" - "strings" "time" "github.com/nirmata/kyverno/pkg/info" @@ -11,7 +10,7 @@ import ( "github.com/nirmata/kyverno/pkg/engine" "github.com/golang/glog" - types "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" + v1alpha1 "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" lister "github.com/nirmata/kyverno/pkg/client/listers/policy/v1alpha1" client "github.com/nirmata/kyverno/pkg/dclient" "github.com/nirmata/kyverno/pkg/event" @@ -63,13 +62,13 @@ func (pc *PolicyController) createPolicyHandler(resource interface{}) { } func (pc *PolicyController) updatePolicyHandler(oldResource, newResource interface{}) { - newPolicy := newResource.(*types.Policy) - oldPolicy := oldResource.(*types.Policy) - newPolicy.Status = types.Status{} - oldPolicy.Status = types.Status{} + newPolicy := newResource.(*v1alpha1.Policy) + oldPolicy := oldResource.(*v1alpha1.Policy) + newPolicy.Status = v1alpha1.Status{} + oldPolicy.Status = v1alpha1.Status{} newPolicy.ResourceVersion = "" oldPolicy.ResourceVersion = "" - if reflect.DeepEqual(newPolicy.ResourceVersion, oldPolicy.ResourceVersion) { + if reflect.DeepEqual(newPolicy, oldPolicy) { return } pc.enqueuePolicy(newResource) @@ -185,7 +184,7 @@ func (pc *PolicyController) syncHandler(obj interface{}) error { //TODO: processPolicy glog.Infof("process policy %s on existing resources", policy.GetName()) policyInfos := engine.ProcessExisting(pc.client, policy) - events, violations := createEventsAndViolations(pc.eventController, policyInfos) + events, violations := pc.createEventsAndViolations(policyInfos) pc.eventController.Add(events...) err = pc.violationBuilder.Add(violations...) if err != nil { @@ -194,25 +193,34 @@ func (pc *PolicyController) syncHandler(obj interface{}) error { return nil } -func createEventsAndViolations(eventController event.Generator, policyInfos []*info.PolicyInfo) ([]*event.Info, []*violation.Info) { +func (pc *PolicyController) createEventsAndViolations(policyInfos []*info.PolicyInfo) ([]*event.Info, []*violation.Info) { events := []*event.Info{} violations := []*violation.Info{} // Create events from the policyInfo for _, policyInfo := range policyInfos { - fruleNames := []string{} + frules := []v1alpha1.FailedRule{} sruleNames := []string{} for _, rule := range policyInfo.Rules { if !rule.IsSuccessful() { e := &event.Info{} - fruleNames = append(fruleNames, rule.Name) + frule := v1alpha1.FailedRule{Name: rule.Name} switch rule.RuleType { case info.Mutation, info.Validation, info.Generation: // Events e = event.NewEvent(policyInfo.RKind, policyInfo.RNamespace, policyInfo.RName, event.PolicyViolation, event.FProcessRule, rule.Name, policyInfo.Name) + switch rule.RuleType { + case info.Mutation: + frule.Type = info.Mutation.String() + case info.Validation: + frule.Type = info.Validation.String() + case info.Generation: + frule.Type = info.Generation.String() + } default: glog.Info("Unsupported Rule type") } + frules = append(frules, frule) events = append(events, e) } else { sruleNames = append(sruleNames, rule.Name) @@ -222,14 +230,20 @@ func createEventsAndViolations(eventController event.Generator, policyInfos []*i if !policyInfo.IsSuccessful() { // Event // list of failed rules : ruleNames - e := event.NewEvent("Policy", "", policyInfo.Name, event.PolicyViolation, event.FResourcePolcy, policyInfo.RNamespace+"/"+policyInfo.RName, strings.Join(fruleNames, ";")) + + e := event.NewEvent("Policy", "", policyInfo.Name, event.PolicyViolation, event.FResourcePolcy, policyInfo.RNamespace+"/"+policyInfo.RName, concatFailedRules(frules)) events = append(events, e) // Violation // TODO: Violation is currently create at policy, level not resource level // As we create violation, we check if the - v := violation.BuldNewViolation(policyInfo.Name, policyInfo.RKind, policyInfo.RNamespace, policyInfo.RName, event.PolicyViolation.String(), fruleNames) + v := violation.BuldNewViolation(policyInfo.Name, policyInfo.RKind, policyInfo.RNamespace, policyInfo.RName, event.PolicyViolation.String(), frules) violations = append(violations, v) + } else { + // clean up violations + pc.violationBuilder.RemoveInactiveViolation(policyInfo.Name, policyInfo.RKind, policyInfo.RNamespace, policyInfo.RName, info.Mutation) + pc.violationBuilder.RemoveInactiveViolation(policyInfo.Name, policyInfo.RKind, policyInfo.RNamespace, policyInfo.RName, info.Validation) } + // else { // // Policy was processed succesfully // e := event.NewEvent("Policy", "", policyInfo.Name, event.PolicyApplied, event.SPolicyApply, policyInfo.Name) diff --git a/pkg/controller/utils.go b/pkg/controller/utils.go index b5eb6de4d6..f353d1408c 100644 --- a/pkg/controller/utils.go +++ b/pkg/controller/utils.go @@ -1,7 +1,21 @@ package controller +import ( + "bytes" + + v1alpha1 "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" +) + const policyWorkQueueName = "policyworkqueue" const policyWorkQueueRetryLimit = 5 const policyControllerWorkerCount = 2 + +func concatFailedRules(frules []v1alpha1.FailedRule) string { + var buffer bytes.Buffer + for _, frule := range frules { + buffer.WriteString(frule.Name + ";") + } + return buffer.String() +} diff --git a/pkg/violation/builder.go b/pkg/violation/builder.go index ff48c55e8f..f3da3833da 100644 --- a/pkg/violation/builder.go +++ b/pkg/violation/builder.go @@ -2,24 +2,27 @@ package violation import ( "errors" + "fmt" "reflect" "github.com/golang/glog" - types "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" - v1alpha1 "github.com/nirmata/kyverno/pkg/client/listers/policy/v1alpha1" + v1alpha1 "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" + lister "github.com/nirmata/kyverno/pkg/client/listers/policy/v1alpha1" client "github.com/nirmata/kyverno/pkg/dclient" event "github.com/nirmata/kyverno/pkg/event" + "github.com/nirmata/kyverno/pkg/info" "github.com/nirmata/kyverno/pkg/sharedinformer" ) //Generator to generate policy violation type Generator interface { Add(infos ...*Info) error + RemoveInactiveViolation(policy, rKind, rNs, rName string, ruleType info.RuleType) error } type builder struct { client *client.Client - policyLister v1alpha1.PolicyLister + policyLister lister.PolicyLister eventBuilder event.Generator } @@ -155,7 +158,7 @@ func (b *builder) isActive(kind, rname, rnamespace string) (bool, error) { //NewViolation return new policy violation func NewViolation(reason event.Reason, policyName, kind, rname, rnamespace, msg string) *Info { return &Info{Policy: policyName, - Violation: types.Violation{ + Violation: v1alpha1.Violation{ Kind: kind, Name: rname, Namespace: rnamespace, @@ -178,29 +181,61 @@ func NewViolation(reason event.Reason, policyName, kind, rname, rnamespace, msg // } // } // Build a new Violation -func BuldNewViolation(pName string, rKind string, rNs string, rName string, reason string, rules []string) *Info { +func BuldNewViolation(pName string, rKind string, rNs string, rName string, reason string, frules []v1alpha1.FailedRule) *Info { return &Info{ Policy: pName, - Violation: types.Violation{ + Violation: v1alpha1.Violation{ Kind: rKind, Namespace: rNs, Name: rName, Reason: reason, - Rules: rules, + Rules: frules, }, } } -func isRuleNamesEqual(currRules []interface{}, newRules []string) bool { +func removeRuleTypes(currRules []interface{}, ruleType info.RuleType) ([]interface{}, error) { + //rules := []v1alpha1.FailedRule{} + var rules []interface{} + // removedRuleCount := 0 + for _, r := range currRules { + glog.Info(reflect.TypeOf(r)) + rfule, ok := r.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("incorrect type") + } + glog.Info(reflect.TypeOf(rfule["type"])) + rtype, ok := rfule["type"].(string) + if !ok { + return nil, fmt.Errorf("incorrect type") + } + name, ok := rfule["name"].(string) + if !ok { + return nil, fmt.Errorf("incorrect type") + } + if rtype != ruleType.String() { + rules = append(rules, v1alpha1.FailedRule{Name: name, Type: rtype}) + } + } + return rules, nil +} + +func isRuleNamesEqual(currRules []interface{}, newRules []v1alpha1.FailedRule) bool { if len(currRules) != len(newRules) { return false } for i, r := range currRules { - name, ok := r.(string) + glog.Info(reflect.TypeOf(r)) + rfule, ok := r.(map[string]interface{}) if !ok { return false } - if name != newRules[i] { + glog.Info(reflect.TypeOf(rfule["name"])) + name, ok := rfule["name"].(string) + if !ok { + return false + } + if name != newRules[i].Name { return false } } @@ -208,10 +243,76 @@ func isRuleNamesEqual(currRules []interface{}, newRules []string) bool { } //RemoveViolation will remove the violation for the resource if there was one -func RemoveViolation(policy *types.Policy, rKind string, rNs string, rName string) { +func (b *builder) RemoveInactiveViolation(policy, rKind, rNs, rName string, ruleType info.RuleType) error { // Remove the pair from map - if policy.Status.Violations != nil { - glog.Infof("Cleaning up violalation for policy %s, resource %s/%s/%s", policy.Name, rKind, rNs, rName) - delete(policy.Status.Violations, BuildKey(rKind, rNs, rName)) + statusMap := map[string]interface{}{} + currVs := map[string]interface{}{} + // Get the policy + p, err := b.client.GetResource("Policy", "", policy, "status") + if err != nil { + glog.Infof("policy %s not found", policy) + return err } + unstr := p.UnstructuredContent() + + // check if "status" field exists + status, ok := unstr["status"] + if ok { + // status is already present then we append violations + if statusMap, ok = status.(map[string]interface{}); !ok { + return errors.New("Unable to parse status subresource") + } + violations, ok := statusMap["violations"] + if !ok { + glog.Info("violation not present") + } + glog.Info(reflect.TypeOf(violations)) + if currVs, ok = violations.(map[string]interface{}); !ok { + return errors.New("Unable to parse violations") + } + currV, ok := currVs[BuildKey(rKind, rNs, rName)] + if !ok { + // No Violation present + return nil + } + glog.Info(reflect.TypeOf(currV)) + v, ok := currV.(map[string]interface{}) + if !ok { + glog.Info("type not matching") + } + // get rules + rules, ok := v["rules"] + if !ok { + glog.Info("rules not found") + } + glog.Info(reflect.TypeOf(rules)) + rs, ok := rules.([]interface{}) + if !ok { + glog.Info("type not matching") + } + // Remove rules of defined type + newrs, err := removeRuleTypes(rs, ruleType) + if err != nil { + glog.Info(err) + } + if newrs == nil { + // all records are removed and is empty + glog.Info("can remove the record") + delete(currVs, BuildKey(rKind, rNs, rName)) + } else { + v["rules"] = newrs + // update the violation with new rule + currVs[BuildKey(rKind, rNs, rName)] = v + } + // update violations + statusMap["violations"] = currVs + // update status + unstr["status"] = statusMap + p.SetUnstructuredContent(unstr) + _, err = b.client.UpdateStatusResource("Policy", "", p, false) + if err != nil { + return err + } + } + return nil } diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index 29b9830629..25c2544c90 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -33,11 +33,12 @@ const policyKind = "Policy" // WebhookServer contains configured TLS server with MutationWebhook. // MutationWebhook gets policies from policyController and takes control of the cluster with kubeclient. type WebhookServer struct { - server http.Server - client *client.Client - policyLister v1alpha1.PolicyLister - eventController event.Generator - filterKinds []string + server http.Server + client *client.Client + policyLister v1alpha1.PolicyLister + eventController event.Generator + violationBuilder violation.Generator + filterKinds []string } // NewWebhookServer creates new instance of WebhookServer accordingly to given configuration @@ -47,6 +48,7 @@ func NewWebhookServer( tlsPair *tlsutils.TlsPemPair, shareInformer sharedinformer.PolicyInformer, eventController event.Generator, + violationBuilder violation.Generator, filterKinds []string) (*WebhookServer, error) { if tlsPair == nil { @@ -61,10 +63,11 @@ func NewWebhookServer( tlsConfig.Certificates = []tls.Certificate{pair} ws := &WebhookServer{ - client: client, - policyLister: shareInformer.GetLister(), - eventController: eventController, - filterKinds: parseKinds(filterKinds), + client: client, + policyLister: shareInformer.GetLister(), + eventController: eventController, + violationBuilder: violationBuilder, + filterKinds: parseKinds(filterKinds), } mux := http.NewServeMux() mux.HandleFunc(config.MutatingWebhookServicePath, ws.serve) @@ -184,9 +187,12 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be glog.Warning(r.Msgs) } } else { - fmt.Println("cleanup") // CleanUp Violations if exists - violation.RemoveViolation(policy, request.Kind.Kind, rns, rname) + err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Mutation) + if err != nil { + glog.Info(err) + } + if len(policyPatches) > 0 { allPatches = append(allPatches, policyPatches...) glog.Infof("Mutation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) @@ -264,7 +270,7 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 glog.V(3).Infof("Handling validation for Kind=%s, Namespace=%s Name=%s UID=%s patchOperation=%s", request.Kind.Kind, rns, rname, request.UID, request.Operation) - glog.Infof("Validating resource with policy %s with %d rules", policy.ObjectMeta.Name, len(policy.Spec.Rules)) + glog.Infof("Validating resource %s/%s/%s with policy %s with %d rules", rkind, rns, rname, policy.ObjectMeta.Name, len(policy.Spec.Rules)) ruleInfos, err := engine.Validate(*policy, request.Object.Raw, request.Kind) if err != nil { // This is not policy error @@ -282,7 +288,10 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 } } else { // CleanUp Violations if exists - violation.RemoveViolation(policy, request.Kind.Kind, rns, rname) + err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Validation) + if err != nil { + glog.Info(err) + } if len(ruleInfos) > 0 { glog.Infof("Validation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) From 11da52c32971a2e7cd7158bb32447983b9d97b12 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Mon, 15 Jul 2019 15:30:28 -0700 Subject: [PATCH 03/24] add mode(blockChanges,reportViolation) CRD spec --- definitions/install.yaml | 11 ++++++++++- definitions/install_debug.yaml | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/definitions/install.yaml b/definitions/install.yaml index 8e83c014cd..34f3ba78c5 100644 --- a/definitions/install.yaml +++ b/definitions/install.yaml @@ -21,7 +21,16 @@ spec: spec: required: - rules + # set as required, as we cannot set default yet. check below for more details + - mode properties: + mode: + type: string + # default can only be set if CustomResourceDefaulting feature gate is enabled + # default: blockChanges + enum: + - blockChanges + - reportViolation rules: type: array items: @@ -177,7 +186,7 @@ spec: serviceAccountName: kyverno-service-account containers: - name: kyverno - image: nirmata/kyverno:latest + image: nirmata/kyverno:dev-testing args: ["--filterKind","Nodes,Events,APIService,SubjectAccessReview"] ports: - containerPort: 443 diff --git a/definitions/install_debug.yaml b/definitions/install_debug.yaml index 313c348b65..f136f80978 100644 --- a/definitions/install_debug.yaml +++ b/definitions/install_debug.yaml @@ -21,7 +21,16 @@ spec: spec: required: - rules + # set as required, as we cannot set default yet. check below for more details + - mode properties: + mode: + type: string + # default can only be set if CustomResourceDefaulting feature gate is enabled + # default: blockChanges + enum: + - blockChanges + - reportViolation rules: type: array items: From abfe176bacbe62a324343b252bb5af4e79c6672b Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Mon, 15 Jul 2019 15:55:53 -0700 Subject: [PATCH 04/24] add helper functions --- pkg/info/info.go | 22 ++++++++++++++ pkg/webhooks/server.go | 68 +++++++++++++++--------------------------- 2 files changed, 46 insertions(+), 44 deletions(-) diff --git a/pkg/info/info.go b/pkg/info/info.go index 3f839bd098..5987678958 100644 --- a/pkg/info/info.go +++ b/pkg/info/info.go @@ -37,6 +37,28 @@ func (pi *PolicyInfo) IsSuccessful() bool { return pi.success } +// SuccessfulRules returns list of successful rule names +func (pi *PolicyInfo) SuccessfulRules() []string { + var rules []string + for _, r := range pi.Rules { + if r.IsSuccessful() { + rules = append(rules, r.Name) + } + } + return rules +} + +// FailedRules returns list of failed rule names +func (pi *PolicyInfo) FailedRules() []string { + var rules []string + for _, r := range pi.Rules { + if !r.IsSuccessful() { + rules = append(rules, r.Name) + } + } + return rules +} + //ErrorRules returns error msgs from all rule func (pi *PolicyInfo) ErrorRules() string { errorMsgs := []string{} diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index 25c2544c90..a80484a493 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -391,59 +391,39 @@ func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool var eventsInfo []*event.Info ok, msg := isAdmSuccesful(policyInfoList) - // create events on operation UPDATE - if onUpdate { - if !ok { - for _, pi := range policyInfoList { - ruleNames := getRuleNames(*pi, false) + // Some policies failed to apply succesfully + if !ok { + for _, pi := range policyInfoList { + rules := pi.FailedRules() + ruleNames := strings.Join(rules, ";") + if !onUpdate { + // CREATE + eventsInfo = append(eventsInfo, + event.NewEvent(policyKind, "", pi.Name, event.RequestBlocked, event.FPolicyApplyBlockCreate, pi.RName, ruleNames)) + + glog.V(3).Infof("Rule(s) %s of policy %s blocked resource creation, error: %s\n", ruleNames, pi.Name, msg) + } else { + // UPDATE eventsInfo = append(eventsInfo, event.NewEvent(pi.RKind, pi.RNamespace, pi.RName, event.RequestBlocked, event.FPolicyApplyBlockUpdate, ruleNames, pi.Name)) - eventsInfo = append(eventsInfo, event.NewEvent(policyKind, "", pi.Name, event.RequestBlocked, event.FPolicyBlockResourceUpdate, pi.RName, ruleNames)) - glog.V(3).Infof("Request blocked events info has prepared for %s/%s and %s/%s\n", policyKind, pi.Name, pi.RKind, pi.RName) } } - return eventsInfo - } + } else { + if !onUpdate { + // All policies were applied succesfully + // CREATE + for _, pi := range policyInfoList { + rules := pi.SuccessfulRules() + ruleNames := strings.Join(rules, ";") + eventsInfo = append(eventsInfo, + event.NewEvent(pi.RKind, pi.RNamespace, pi.RName, event.PolicyApplied, event.SRulesApply, ruleNames, pi.Name)) - // create events on operation CREATE - if ok { - for _, pi := range policyInfoList { - ruleNames := getRuleNames(*pi, true) - - eventsInfo = append(eventsInfo, - event.NewEvent(pi.RKind, pi.RNamespace, pi.RName, event.PolicyApplied, event.SRulesApply, ruleNames, pi.Name)) - - glog.V(3).Infof("Success event info has prepared for %s/%s\n", pi.RKind, pi.RName) + glog.V(3).Infof("Success event info has prepared for %s/%s\n", pi.RKind, pi.RName) + } } - return eventsInfo - } - - for _, pi := range policyInfoList { - ruleNames := getRuleNames(*pi, false) - - eventsInfo = append(eventsInfo, - event.NewEvent(policyKind, "", pi.Name, event.RequestBlocked, event.FPolicyApplyBlockCreate, pi.RName, ruleNames)) - - glog.V(3).Infof("Rule(s) %s of policy %s blocked resource creation, error: %s\n", ruleNames, pi.Name, msg) } return eventsInfo } - -func getRuleNames(policyInfo info.PolicyInfo, onSuccess bool) string { - var ruleNames []string - for _, rule := range policyInfo.Rules { - if onSuccess { - if rule.IsSuccessful() { - ruleNames = append(ruleNames, rule.Name) - } - } else { - if !rule.IsSuccessful() { - ruleNames = append(ruleNames, rule.Name) - } - } - } - return strings.Join(ruleNames, ",") -} From f879375ab39ab5a4d051307cccc060ad37932077 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Mon, 15 Jul 2019 16:02:19 -0700 Subject: [PATCH 05/24] add mode to policy struct --- pkg/apis/policy/v1alpha1/types.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/apis/policy/v1alpha1/types.go b/pkg/apis/policy/v1alpha1/types.go index 2d30a9e181..061341b6d8 100644 --- a/pkg/apis/policy/v1alpha1/types.go +++ b/pkg/apis/policy/v1alpha1/types.go @@ -19,6 +19,7 @@ type Policy struct { // Spec describes policy behavior by its rules type Spec struct { Rules []Rule `json:"rules"` + Mode string `json:"mode"` } // Rule is set of mutation, validation and generation actions From 68a67519907ed973e7c78ed6ef8c0add80363f4f Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Mon, 15 Jul 2019 16:07:56 -0700 Subject: [PATCH 06/24] restructure webhooks pkg --- pkg/webhooks/mutation.go | 89 +++++++++++ pkg/webhooks/policyvalidation.go | 45 ++++++ pkg/webhooks/report.go | 50 ++++++ pkg/webhooks/server.go | 261 ------------------------------- pkg/webhooks/utils.go | 19 +++ pkg/webhooks/validation.go | 95 +++++++++++ 6 files changed, 298 insertions(+), 261 deletions(-) create mode 100644 pkg/webhooks/mutation.go create mode 100644 pkg/webhooks/policyvalidation.go create mode 100644 pkg/webhooks/report.go create mode 100644 pkg/webhooks/validation.go diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go new file mode 100644 index 0000000000..aea8e45853 --- /dev/null +++ b/pkg/webhooks/mutation.go @@ -0,0 +1,89 @@ +package webhooks + +import ( + "github.com/golang/glog" + engine "github.com/nirmata/kyverno/pkg/engine" + "github.com/nirmata/kyverno/pkg/info" + v1beta1 "k8s.io/api/admission/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" +) + +// HandleMutation handles mutating webhook admission request +func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { + + policies, err := ws.policyLister.List(labels.NewSelector()) + if err != nil { + // Unable to connect to policy Lister to access policies + glog.Error("Unable to connect to policy controller to access policies. Mutation Rules are NOT being applied") + glog.Warning(err) + return &v1beta1.AdmissionResponse{ + Allowed: true, + } + } + + var allPatches [][]byte + policyInfos := []*info.PolicyInfo{} + for _, policy := range policies { + // check if policy has a rule for the admission request kind + if !StringInSlice(request.Kind.Kind, getApplicableKindsForPolicy(policy)) { + continue + } + rname := engine.ParseNameFromObject(request.Object.Raw) + rns := engine.ParseNamespaceFromObject(request.Object.Raw) + rkind := engine.ParseKindFromObject(request.Object.Raw) + policyInfo := info.NewPolicyInfo(policy.Name, + rkind, + rname, + rns) + + glog.V(3).Infof("Handling mutation for Kind=%s, Namespace=%s Name=%s UID=%s patchOperation=%s", + request.Kind.Kind, rns, rname, request.UID, request.Operation) + + glog.Infof("Applying policy %s with %d rules\n", policy.ObjectMeta.Name, len(policy.Spec.Rules)) + + policyPatches, ruleInfos := engine.Mutate(*policy, request.Object.Raw, request.Kind) + + policyInfo.AddRuleInfos(ruleInfos) + + if !policyInfo.IsSuccessful() { + glog.Infof("Failed to apply policy %s on resource %s/%s", policy.Name, rname, rns) + for _, r := range ruleInfos { + glog.Warning(r.Msgs) + } + } else { + // CleanUp Violations if exists + err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Mutation) + if err != nil { + glog.Info(err) + } + + if len(policyPatches) > 0 { + allPatches = append(allPatches, policyPatches...) + glog.Infof("Mutation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) + } + } + policyInfos = append(policyInfos, policyInfo) + } + + if len(allPatches) > 0 { + eventsInfo := newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update)) + ws.eventController.Add(eventsInfo...) + } + + ok, msg := isAdmSuccesful(policyInfos) + if ok { + patchType := v1beta1.PatchTypeJSONPatch + return &v1beta1.AdmissionResponse{ + Allowed: true, + Patch: engine.JoinPatches(allPatches), + PatchType: &patchType, + } + } + return &v1beta1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Message: msg, + }, + } +} diff --git a/pkg/webhooks/policyvalidation.go b/pkg/webhooks/policyvalidation.go new file mode 100644 index 0000000000..873e7e5bfa --- /dev/null +++ b/pkg/webhooks/policyvalidation.go @@ -0,0 +1,45 @@ +package webhooks + +import ( + "encoding/json" + "fmt" + + "github.com/golang/glog" + policyv1 "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" + "github.com/nirmata/kyverno/pkg/utils" + v1beta1 "k8s.io/api/admission/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +//HandlePolicyValidation performs the validation check on policy resource +func (ws *WebhookServer) HandlePolicyValidation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { + return ws.validateUniqueRuleName(request.Object.Raw) +} + +// Verify if the Rule names are unique within a policy +func (ws *WebhookServer) validateUniqueRuleName(rawPolicy []byte) *v1beta1.AdmissionResponse { + var policy *policyv1.Policy + var ruleNames []string + + json.Unmarshal(rawPolicy, &policy) + + for _, rule := range policy.Spec.Rules { + if utils.Contains(ruleNames, rule.Name) { + msg := fmt.Sprintf(`The policy "%s" is invalid: duplicate rule name: "%s"`, policy.Name, rule.Name) + glog.Errorln(msg) + + return &v1beta1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Message: msg, + }, + } + } + ruleNames = append(ruleNames, rule.Name) + } + + glog.V(3).Infof("Policy validation passed") + return &v1beta1.AdmissionResponse{ + Allowed: true, + } +} diff --git a/pkg/webhooks/report.go b/pkg/webhooks/report.go new file mode 100644 index 0000000000..8f9e57a8ac --- /dev/null +++ b/pkg/webhooks/report.go @@ -0,0 +1,50 @@ +package webhooks + +import ( + "strings" + + "github.com/golang/glog" + "github.com/nirmata/kyverno/pkg/event" + "github.com/nirmata/kyverno/pkg/info" +) + +func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool) []*event.Info { + var eventsInfo []*event.Info + + ok, msg := isAdmSuccesful(policyInfoList) + // Some policies failed to apply succesfully + if !ok { + for _, pi := range policyInfoList { + rules := pi.FailedRules() + ruleNames := strings.Join(rules, ";") + if !onUpdate { + // CREATE + eventsInfo = append(eventsInfo, + event.NewEvent(policyKind, "", pi.Name, event.RequestBlocked, event.FPolicyApplyBlockCreate, pi.RName, ruleNames)) + + glog.V(3).Infof("Rule(s) %s of policy %s blocked resource creation, error: %s\n", ruleNames, pi.Name, msg) + } else { + // UPDATE + eventsInfo = append(eventsInfo, + event.NewEvent(pi.RKind, pi.RNamespace, pi.RName, event.RequestBlocked, event.FPolicyApplyBlockUpdate, ruleNames, pi.Name)) + eventsInfo = append(eventsInfo, + event.NewEvent(policyKind, "", pi.Name, event.RequestBlocked, event.FPolicyBlockResourceUpdate, pi.RName, ruleNames)) + glog.V(3).Infof("Request blocked events info has prepared for %s/%s and %s/%s\n", policyKind, pi.Name, pi.RKind, pi.RName) + } + } + } else { + if !onUpdate { + // All policies were applied succesfully + // CREATE + for _, pi := range policyInfoList { + rules := pi.SuccessfulRules() + ruleNames := strings.Join(rules, ";") + eventsInfo = append(eventsInfo, + event.NewEvent(pi.RKind, pi.RNamespace, pi.RName, event.PolicyApplied, event.SRulesApply, ruleNames, pi.Name)) + + glog.V(3).Infof("Success event info has prepared for %s/%s\n", pi.RKind, pi.RName) + } + } + } + return eventsInfo +} diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index a80484a493..a2b656d558 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -8,28 +8,19 @@ import ( "fmt" "io/ioutil" "net/http" - "strings" "time" "github.com/golang/glog" - policyv1 "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" "github.com/nirmata/kyverno/pkg/client/listers/policy/v1alpha1" "github.com/nirmata/kyverno/pkg/config" client "github.com/nirmata/kyverno/pkg/dclient" - engine "github.com/nirmata/kyverno/pkg/engine" "github.com/nirmata/kyverno/pkg/event" - "github.com/nirmata/kyverno/pkg/info" "github.com/nirmata/kyverno/pkg/sharedinformer" tlsutils "github.com/nirmata/kyverno/pkg/tls" - "github.com/nirmata/kyverno/pkg/utils" "github.com/nirmata/kyverno/pkg/violation" v1beta1 "k8s.io/api/admission/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" ) -const policyKind = "Policy" - // WebhookServer contains configured TLS server with MutationWebhook. // MutationWebhook gets policies from policyController and takes control of the cluster with kubeclient. type WebhookServer struct { @@ -144,185 +135,6 @@ func (ws *WebhookServer) Stop() { } } -// HandleMutation handles mutating webhook admission request -func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { - - policies, err := ws.policyLister.List(labels.NewSelector()) - if err != nil { - // Unable to connect to policy Lister to access policies - glog.Error("Unable to connect to policy controller to access policies. Mutation Rules are NOT being applied") - glog.Warning(err) - return &v1beta1.AdmissionResponse{ - Allowed: true, - } - } - - var allPatches [][]byte - policyInfos := []*info.PolicyInfo{} - for _, policy := range policies { - // check if policy has a rule for the admission request kind - if !StringInSlice(request.Kind.Kind, getApplicableKindsForPolicy(policy)) { - continue - } - rname := engine.ParseNameFromObject(request.Object.Raw) - rns := engine.ParseNamespaceFromObject(request.Object.Raw) - rkind := engine.ParseKindFromObject(request.Object.Raw) - policyInfo := info.NewPolicyInfo(policy.Name, - rkind, - rname, - rns) - - glog.V(3).Infof("Handling mutation for Kind=%s, Namespace=%s Name=%s UID=%s patchOperation=%s", - request.Kind.Kind, rns, rname, request.UID, request.Operation) - - glog.Infof("Applying policy %s with %d rules\n", policy.ObjectMeta.Name, len(policy.Spec.Rules)) - - policyPatches, ruleInfos := engine.Mutate(*policy, request.Object.Raw, request.Kind) - - policyInfo.AddRuleInfos(ruleInfos) - - if !policyInfo.IsSuccessful() { - glog.Infof("Failed to apply policy %s on resource %s/%s", policy.Name, rname, rns) - for _, r := range ruleInfos { - glog.Warning(r.Msgs) - } - } else { - // CleanUp Violations if exists - err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Mutation) - if err != nil { - glog.Info(err) - } - - if len(policyPatches) > 0 { - allPatches = append(allPatches, policyPatches...) - glog.Infof("Mutation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) - } - } - policyInfos = append(policyInfos, policyInfo) - } - - if len(allPatches) > 0 { - eventsInfo := newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update)) - ws.eventController.Add(eventsInfo...) - } - - ok, msg := isAdmSuccesful(policyInfos) - if ok { - patchType := v1beta1.PatchTypeJSONPatch - return &v1beta1.AdmissionResponse{ - Allowed: true, - Patch: engine.JoinPatches(allPatches), - PatchType: &patchType, - } - } - return &v1beta1.AdmissionResponse{ - Allowed: false, - Result: &metav1.Status{ - Message: msg, - }, - } -} - -func isAdmSuccesful(policyInfos []*info.PolicyInfo) (bool, string) { - var admSuccess = true - var errMsgs []string - for _, pi := range policyInfos { - if !pi.IsSuccessful() { - admSuccess = false - errMsgs = append(errMsgs, fmt.Sprintf("\nPolicy %s failed with following rules", pi.Name)) - // Get the error rules - errorRules := pi.ErrorRules() - errMsgs = append(errMsgs, errorRules) - } - } - return admSuccess, strings.Join(errMsgs, ";") -} - -// HandleValidation handles validating webhook admission request -// If there are no errors in validating rule we apply generation rules -func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { - policyInfos := []*info.PolicyInfo{} - - policies, err := ws.policyLister.List(labels.NewSelector()) - if err != nil { - // Unable to connect to policy Lister to access policies - glog.Error("Unable to connect to policy controller to access policies. Validation Rules are NOT being applied") - glog.Warning(err) - return &v1beta1.AdmissionResponse{ - Allowed: true, - } - } - - for _, policy := range policies { - - if !StringInSlice(request.Kind.Kind, getApplicableKindsForPolicy(policy)) { - continue - } - rname := engine.ParseNameFromObject(request.Object.Raw) - rns := engine.ParseNamespaceFromObject(request.Object.Raw) - rkind := engine.ParseKindFromObject(request.Object.Raw) - - policyInfo := info.NewPolicyInfo(policy.Name, - rkind, - rname, - rns) - - glog.V(3).Infof("Handling validation for Kind=%s, Namespace=%s Name=%s UID=%s patchOperation=%s", - request.Kind.Kind, rns, rname, request.UID, request.Operation) - - glog.Infof("Validating resource %s/%s/%s with policy %s with %d rules", rkind, rns, rname, policy.ObjectMeta.Name, len(policy.Spec.Rules)) - ruleInfos, err := engine.Validate(*policy, request.Object.Raw, request.Kind) - if err != nil { - // This is not policy error - // but if unable to parse request raw resource - // TODO : create event ? dont think so - glog.Error(err) - continue - } - policyInfo.AddRuleInfos(ruleInfos) - - if !policyInfo.IsSuccessful() { - glog.Infof("Failed to apply policy %s on resource %s/%s", policy.Name, rname, rns) - for _, r := range ruleInfos { - glog.Warning(r.Msgs) - } - } else { - // CleanUp Violations if exists - err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Validation) - if err != nil { - glog.Info(err) - } - - if len(ruleInfos) > 0 { - glog.Infof("Validation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) - } - } - policyInfos = append(policyInfos, policyInfo) - } - - if len(policyInfos) > 0 && len(policyInfos[0].Rules) != 0 { - eventsInfo := newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update)) - ws.eventController.Add(eventsInfo...) - - } - - // If Validation fails then reject the request - ok, msg := isAdmSuccesful(policyInfos) - if !ok { - return &v1beta1.AdmissionResponse{ - Allowed: false, - Result: &metav1.Status{ - Message: msg, - }, - } - } - - return &v1beta1.AdmissionResponse{ - Allowed: true, - } - // Generation rules applied via generation controller -} - // bodyToAdmissionReview creates AdmissionReview object from request body // Answers to the http.ResponseWriter if request is not valid func (ws *WebhookServer) bodyToAdmissionReview(request *http.Request, writer http.ResponseWriter) *v1beta1.AdmissionReview { @@ -354,76 +166,3 @@ func (ws *WebhookServer) bodyToAdmissionReview(request *http.Request, writer htt return admissionReview } - -//HandlePolicyValidation performs the validation check on policy resource -func (ws *WebhookServer) HandlePolicyValidation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { - return ws.validateUniqueRuleName(request.Object.Raw) -} - -func (ws *WebhookServer) validateUniqueRuleName(rawPolicy []byte) *v1beta1.AdmissionResponse { - var policy *policyv1.Policy - var ruleNames []string - - json.Unmarshal(rawPolicy, &policy) - - for _, rule := range policy.Spec.Rules { - if utils.Contains(ruleNames, rule.Name) { - msg := fmt.Sprintf(`The policy "%s" is invalid: duplicate rule name: "%s"`, policy.Name, rule.Name) - glog.Errorln(msg) - - return &v1beta1.AdmissionResponse{ - Allowed: false, - Result: &metav1.Status{ - Message: msg, - }, - } - } - ruleNames = append(ruleNames, rule.Name) - } - - glog.V(3).Infof("Policy validation passed") - return &v1beta1.AdmissionResponse{ - Allowed: true, - } -} - -func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool) []*event.Info { - var eventsInfo []*event.Info - - ok, msg := isAdmSuccesful(policyInfoList) - // Some policies failed to apply succesfully - if !ok { - for _, pi := range policyInfoList { - rules := pi.FailedRules() - ruleNames := strings.Join(rules, ";") - if !onUpdate { - // CREATE - eventsInfo = append(eventsInfo, - event.NewEvent(policyKind, "", pi.Name, event.RequestBlocked, event.FPolicyApplyBlockCreate, pi.RName, ruleNames)) - - glog.V(3).Infof("Rule(s) %s of policy %s blocked resource creation, error: %s\n", ruleNames, pi.Name, msg) - } else { - // UPDATE - eventsInfo = append(eventsInfo, - event.NewEvent(pi.RKind, pi.RNamespace, pi.RName, event.RequestBlocked, event.FPolicyApplyBlockUpdate, ruleNames, pi.Name)) - eventsInfo = append(eventsInfo, - event.NewEvent(policyKind, "", pi.Name, event.RequestBlocked, event.FPolicyBlockResourceUpdate, pi.RName, ruleNames)) - glog.V(3).Infof("Request blocked events info has prepared for %s/%s and %s/%s\n", policyKind, pi.Name, pi.RKind, pi.RName) - } - } - } else { - if !onUpdate { - // All policies were applied succesfully - // CREATE - for _, pi := range policyInfoList { - rules := pi.SuccessfulRules() - ruleNames := strings.Join(rules, ";") - eventsInfo = append(eventsInfo, - event.NewEvent(pi.RKind, pi.RNamespace, pi.RName, event.PolicyApplied, event.SRulesApply, ruleNames, pi.Name)) - - glog.V(3).Infof("Success event info has prepared for %s/%s\n", pi.RKind, pi.RName) - } - } - } - return eventsInfo -} diff --git a/pkg/webhooks/utils.go b/pkg/webhooks/utils.go index 116c06053a..97a2f024d2 100644 --- a/pkg/webhooks/utils.go +++ b/pkg/webhooks/utils.go @@ -1,11 +1,30 @@ package webhooks import ( + "fmt" "strings" "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" + "github.com/nirmata/kyverno/pkg/info" ) +const policyKind = "Policy" + +func isAdmSuccesful(policyInfos []*info.PolicyInfo) (bool, string) { + var admSuccess = true + var errMsgs []string + for _, pi := range policyInfos { + if !pi.IsSuccessful() { + admSuccess = false + errMsgs = append(errMsgs, fmt.Sprintf("\nPolicy %s failed with following rules", pi.Name)) + // Get the error rules + errorRules := pi.ErrorRules() + errMsgs = append(errMsgs, errorRules) + } + } + return admSuccess, strings.Join(errMsgs, ";") +} + //StringInSlice checks if string is present in slice of strings func StringInSlice(kind string, list []string) bool { for _, b := range list { diff --git a/pkg/webhooks/validation.go b/pkg/webhooks/validation.go new file mode 100644 index 0000000000..9d596f1a95 --- /dev/null +++ b/pkg/webhooks/validation.go @@ -0,0 +1,95 @@ +package webhooks + +import ( + "github.com/golang/glog" + engine "github.com/nirmata/kyverno/pkg/engine" + "github.com/nirmata/kyverno/pkg/info" + v1beta1 "k8s.io/api/admission/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" +) + +// HandleValidation handles validating webhook admission request +// If there are no errors in validating rule we apply generation rules +func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { + policyInfos := []*info.PolicyInfo{} + + policies, err := ws.policyLister.List(labels.NewSelector()) + if err != nil { + // Unable to connect to policy Lister to access policies + glog.Error("Unable to connect to policy controller to access policies. Validation Rules are NOT being applied") + glog.Warning(err) + return &v1beta1.AdmissionResponse{ + Allowed: true, + } + } + + for _, policy := range policies { + + if !StringInSlice(request.Kind.Kind, getApplicableKindsForPolicy(policy)) { + continue + } + rname := engine.ParseNameFromObject(request.Object.Raw) + rns := engine.ParseNamespaceFromObject(request.Object.Raw) + rkind := engine.ParseKindFromObject(request.Object.Raw) + + policyInfo := info.NewPolicyInfo(policy.Name, + rkind, + rname, + rns) + + glog.V(3).Infof("Handling validation for Kind=%s, Namespace=%s Name=%s UID=%s patchOperation=%s", + request.Kind.Kind, rns, rname, request.UID, request.Operation) + + glog.Infof("Validating resource %s/%s/%s with policy %s with %d rules", rkind, rns, rname, policy.ObjectMeta.Name, len(policy.Spec.Rules)) + ruleInfos, err := engine.Validate(*policy, request.Object.Raw, request.Kind) + if err != nil { + // This is not policy error + // but if unable to parse request raw resource + // TODO : create event ? dont think so + glog.Error(err) + continue + } + policyInfo.AddRuleInfos(ruleInfos) + + if !policyInfo.IsSuccessful() { + glog.Infof("Failed to apply policy %s on resource %s/%s", policy.Name, rname, rns) + for _, r := range ruleInfos { + glog.Warning(r.Msgs) + } + } else { + // CleanUp Violations if exists + err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Validation) + if err != nil { + glog.Info(err) + } + + if len(ruleInfos) > 0 { + glog.Infof("Validation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) + } + } + policyInfos = append(policyInfos, policyInfo) + } + + if len(policyInfos) > 0 && len(policyInfos[0].Rules) != 0 { + eventsInfo := newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update)) + ws.eventController.Add(eventsInfo...) + + } + + // If Validation fails then reject the request + ok, msg := isAdmSuccesful(policyInfos) + if !ok { + return &v1beta1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Message: msg, + }, + } + } + + return &v1beta1.AdmissionResponse{ + Allowed: true, + } + // Generation rules applied via generation controller +} From f47910da5384f556bf0fc55f53f2ee2277c11d6d Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Mon, 15 Jul 2019 19:14:42 -0700 Subject: [PATCH 07/24] update flag & support ValidationFailureAction flag --- pkg/apis/policy/v1alpha1/types.go | 4 ++-- pkg/engine/engine.go | 2 +- pkg/gencontroller/generation.go | 3 ++- pkg/info/info.go | 22 +++++++++++++++++++--- pkg/testrunner/test.go | 3 ++- pkg/webhooks/mutation.go | 5 +++-- pkg/webhooks/report.go | 14 +++++++++++--- pkg/webhooks/utils.go | 6 ++++++ pkg/webhooks/validation.go | 19 ++++++++++++++----- 9 files changed, 60 insertions(+), 18 deletions(-) diff --git a/pkg/apis/policy/v1alpha1/types.go b/pkg/apis/policy/v1alpha1/types.go index 061341b6d8..50be3f5b8e 100644 --- a/pkg/apis/policy/v1alpha1/types.go +++ b/pkg/apis/policy/v1alpha1/types.go @@ -18,8 +18,8 @@ type Policy struct { // Spec describes policy behavior by its rules type Spec struct { - Rules []Rule `json:"rules"` - Mode string `json:"mode"` + Rules []Rule `json:"rules"` + ValidationFailureAction string `json:"validationFailureAction"` } // Rule is set of mutation, validation and generation actions diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 4cc94d80e5..29afad4375 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -74,7 +74,7 @@ func ProcessExisting(client *client.Client, policy *types.Policy) []*info.Policy } func applyPolicy(client *client.Client, policy *types.Policy, res *resourceInfo) (*info.PolicyInfo, error) { - policyInfo := info.NewPolicyInfo(policy.Name, res.gvk.Kind, res.resource.GetName(), res.resource.GetNamespace()) + policyInfo := info.NewPolicyInfo(policy.Name, res.gvk.Kind, res.resource.GetName(), res.resource.GetNamespace(), policy.Spec.ValidationFailureAction) glog.Infof("Applying policy %s with %d rules\n", policy.ObjectMeta.Name, len(policy.Spec.Rules)) rawResource, err := res.resource.MarshalJSON() if err != nil { diff --git a/pkg/gencontroller/generation.go b/pkg/gencontroller/generation.go index b8cc3cea2e..f153d2434a 100644 --- a/pkg/gencontroller/generation.go +++ b/pkg/gencontroller/generation.go @@ -59,7 +59,8 @@ func (c *Controller) processPolicy(ns *corev1.Namespace, p *v1alpha1.Policy) { policyInfo := info.NewPolicyInfo(p.Name, "Namespace", ns.Name, - "") // Namespace has no namespace..WOW + "", + p.Spec.ValidationFailureAction) // Namespace has no namespace..WOW ruleInfos := engine.GenerateNew(c.client, p, ns) policyInfo.AddRuleInfos(ruleInfos) diff --git a/pkg/info/info.go b/pkg/info/info.go index 5987678958..b19922a4c7 100644 --- a/pkg/info/info.go +++ b/pkg/info/info.go @@ -3,6 +3,8 @@ package info import ( "fmt" "strings" + + v1alpha1 "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" ) //PolicyInfo defines policy information @@ -16,18 +18,21 @@ type PolicyInfo struct { // Namespace is the ns of resource // empty on non-namespaced resources RNamespace string - Rules []*RuleInfo - success bool + //TODO: add check/enum for types + Mode string // BlockChanges, ReportViolation + Rules []*RuleInfo + success bool } //NewPolicyInfo returns a new policy info -func NewPolicyInfo(policyName string, rKind string, rName string, rNamespace string) *PolicyInfo { +func NewPolicyInfo(policyName, rKind, rName, rNamespace, mode string) *PolicyInfo { return &PolicyInfo{ Name: policyName, RKind: rKind, RName: rName, RNamespace: rNamespace, success: true, // fail to be set explicity + Mode: mode, } } @@ -59,6 +64,17 @@ func (pi *PolicyInfo) FailedRules() []string { return rules } +//GetFailedRules returns the failed rules with rule type +func (pi *PolicyInfo) GetFailedRules() []v1alpha1.FailedRule { + var rules []v1alpha1.FailedRule + for _, r := range pi.Rules { + if !r.IsSuccessful() { + rules = append(rules, v1alpha1.FailedRule{Name: r.Name, Type: r.RuleType.String()}) + } + } + return rules +} + //ErrorRules returns error msgs from all rule func (pi *PolicyInfo) ErrorRules() string { errorMsgs := []string{} diff --git a/pkg/testrunner/test.go b/pkg/testrunner/test.go index f76ba343b1..d9963e7578 100644 --- a/pkg/testrunner/test.go +++ b/pkg/testrunner/test.go @@ -173,7 +173,8 @@ func (t *test) applyPolicy(policy *pt.Policy, policyInfo := info.NewPolicyInfo(policy.Name, rkind, rname, - rns) + rns, + policy.Spec.Mode) // Apply Mutation Rules patches, ruleInfos := engine.Mutate(*policy, rawResource, *tresource.gvk) policyInfo.AddRuleInfos(ruleInfos) diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go index aea8e45853..4038c7d50d 100644 --- a/pkg/webhooks/mutation.go +++ b/pkg/webhooks/mutation.go @@ -35,7 +35,8 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be policyInfo := info.NewPolicyInfo(policy.Name, rkind, rname, - rns) + rns, + policy.Spec.ValidationFailureAction) glog.V(3).Infof("Handling mutation for Kind=%s, Namespace=%s Name=%s UID=%s patchOperation=%s", request.Kind.Kind, rns, rname, request.UID, request.Operation) @@ -67,7 +68,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be } if len(allPatches) > 0 { - eventsInfo := newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update)) + eventsInfo, _ := newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update)) ws.eventController.Add(eventsInfo...) } diff --git a/pkg/webhooks/report.go b/pkg/webhooks/report.go index 8f9e57a8ac..009fd66462 100644 --- a/pkg/webhooks/report.go +++ b/pkg/webhooks/report.go @@ -3,14 +3,16 @@ package webhooks import ( "strings" + "github.com/nirmata/kyverno/pkg/violation" + "github.com/golang/glog" "github.com/nirmata/kyverno/pkg/event" "github.com/nirmata/kyverno/pkg/info" ) -func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool) []*event.Info { +func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool) ([]*event.Info, []*violation.Info) { var eventsInfo []*event.Info - + var violations []*violation.Info ok, msg := isAdmSuccesful(policyInfoList) // Some policies failed to apply succesfully if !ok { @@ -31,6 +33,12 @@ func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool event.NewEvent(policyKind, "", pi.Name, event.RequestBlocked, event.FPolicyBlockResourceUpdate, pi.RName, ruleNames)) glog.V(3).Infof("Request blocked events info has prepared for %s/%s and %s/%s\n", policyKind, pi.Name, pi.RKind, pi.RName) } + // if report flag is set + if pi.Mode == "reportViolation" { + // Create Violations + v := violation.BuldNewViolation(pi.Name, pi.RKind, pi.RNamespace, pi.RName, event.PolicyViolation.String(), pi.GetFailedRules()) + violations = append(violations, v) + } } } else { if !onUpdate { @@ -46,5 +54,5 @@ func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool } } } - return eventsInfo + return eventsInfo, violations } diff --git a/pkg/webhooks/utils.go b/pkg/webhooks/utils.go index 97a2f024d2..ce4da62f02 100644 --- a/pkg/webhooks/utils.go +++ b/pkg/webhooks/utils.go @@ -82,3 +82,9 @@ func getApplicableKindsForPolicy(p *v1alpha1.Policy) []string { } return kinds } + +// Policy Reporting Modes +const ( + BlockChanges = "blockChanges" + ReportViolation = "reportViolation" +) diff --git a/pkg/webhooks/validation.go b/pkg/webhooks/validation.go index 9d596f1a95..1ecc0af02d 100644 --- a/pkg/webhooks/validation.go +++ b/pkg/webhooks/validation.go @@ -3,7 +3,9 @@ package webhooks import ( "github.com/golang/glog" engine "github.com/nirmata/kyverno/pkg/engine" + "github.com/nirmata/kyverno/pkg/event" "github.com/nirmata/kyverno/pkg/info" + "github.com/nirmata/kyverno/pkg/violation" v1beta1 "k8s.io/api/admission/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -13,7 +15,8 @@ import ( // If there are no errors in validating rule we apply generation rules func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { policyInfos := []*info.PolicyInfo{} - + var violations []*violation.Info + var eventsInfo []*event.Info policies, err := ws.policyLister.List(labels.NewSelector()) if err != nil { // Unable to connect to policy Lister to access policies @@ -36,7 +39,8 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 policyInfo := info.NewPolicyInfo(policy.Name, rkind, rname, - rns) + rns, + policy.Spec.ValidationFailureAction) glog.V(3).Infof("Handling validation for Kind=%s, Namespace=%s Name=%s UID=%s patchOperation=%s", request.Kind.Kind, rns, rname, request.UID, request.Operation) @@ -72,14 +76,19 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 } if len(policyInfos) > 0 && len(policyInfos[0].Rules) != 0 { - eventsInfo := newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update)) + eventsInfo, violations = newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update)) + // If the validationFailureAction flag is set "report", + // then we dont block the request and report the violations + ws.violationBuilder.Add(violations...) ws.eventController.Add(eventsInfo...) - } // If Validation fails then reject the request ok, msg := isAdmSuccesful(policyInfos) - if !ok { + // violations are created if "report" flag is set + // and if there are any then we dont bock the resource creation + // Even if one the policy being applied + if !ok && violations == nil { return &v1beta1.AdmissionResponse{ Allowed: false, Result: &metav1.Status{ From 68a48373ae5c4f4ba94ce8fbf900aac4b6a9a551 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Mon, 15 Jul 2019 20:16:06 -0700 Subject: [PATCH 08/24] add delete op to webhook & violation removal --- pkg/violation/builder.go | 48 +++++++++++++++++++++++++++++++++ pkg/webhooks/deleteresource.go | 40 +++++++++++++++++++++++++++ pkg/webhooks/registration.go | 49 +++++++++++++++++++++++----------- pkg/webhooks/server.go | 30 ++++++++++++++++----- 4 files changed, 144 insertions(+), 23 deletions(-) create mode 100644 pkg/webhooks/deleteresource.go diff --git a/pkg/violation/builder.go b/pkg/violation/builder.go index f3da3833da..7956ec065b 100644 --- a/pkg/violation/builder.go +++ b/pkg/violation/builder.go @@ -18,6 +18,7 @@ import ( type Generator interface { Add(infos ...*Info) error RemoveInactiveViolation(policy, rKind, rNs, rName string, ruleType info.RuleType) error + ResourceRemoval(policy, rKind, rNs, rName string) error } type builder struct { @@ -316,3 +317,50 @@ func (b *builder) RemoveInactiveViolation(policy, rKind, rNs, rName string, rule } return nil } + +func (b *builder) ResourceRemoval(policy, rKind, rNs, rName string) error { + // Remove the pair from map + statusMap := map[string]interface{}{} + currVs := map[string]interface{}{} + // Get the policy + p, err := b.client.GetResource("Policy", "", policy, "status") + if err != nil { + glog.Infof("policy %s not found", policy) + return err + } + unstr := p.UnstructuredContent() + // check if "status" field exists + status, ok := unstr["status"] + if ok { + // status is already present then we append violations + if statusMap, ok = status.(map[string]interface{}); !ok { + return errors.New("Unable to parse status subresource") + } + violations, ok := statusMap["violations"] + if !ok { + glog.Info("violation not present") + } + glog.Info(reflect.TypeOf(violations)) + if currVs, ok = violations.(map[string]interface{}); !ok { + return errors.New("Unable to parse violations") + } + _, ok = currVs[BuildKey(rKind, rNs, rName)] + if !ok { + // No Violation for this resource + return nil + } + // remove the pair from the map + delete(currVs, BuildKey(rKind, rNs, rName)) + glog.Info("Removed Violation") + // update violations + statusMap["violations"] = currVs + // update status + unstr["status"] = statusMap + p.SetUnstructuredContent(unstr) + _, err = b.client.UpdateStatusResource("Policy", "", p, false) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/webhooks/deleteresource.go b/pkg/webhooks/deleteresource.go new file mode 100644 index 0000000000..7d5743f33b --- /dev/null +++ b/pkg/webhooks/deleteresource.go @@ -0,0 +1,40 @@ +package webhooks + +import ( + "errors" + + engine "github.com/nirmata/kyverno/pkg/engine" + v1beta1 "k8s.io/api/admission/v1beta1" + "k8s.io/apimachinery/pkg/labels" +) + +func (ws *WebhookServer) removePolicyViolation(request *v1beta1.AdmissionRequest) error { + + // Get the list of policies that apply on the resource + policies, err := ws.policyLister.List(labels.NewSelector()) + if err != nil { + // Unable to connect to policy Lister to access policies + return errors.New("Unable to connect to policy controller to access policies. Clean Up of Policy Violations is not being done") + } + for _, policy := range policies { + // check if policy has a rule for the admission request kind + if !StringInSlice(request.Kind.Kind, getApplicableKindsForPolicy(policy)) { + continue + } + rname := engine.ParseNameFromObject(request.Object.Raw) + rns := engine.ParseNamespaceFromObject(request.Object.Raw) + rkind := engine.ParseKindFromObject(request.Object.Raw) + // check if the resource meets the policy Resource description + for _, rule := range policy.Spec.Rules { + ok := engine.ResourceMeetsDescription(request.Object.Raw, rule.ResourceDescription, request.Kind) + if ok { + // Check if the policy has a violation for this resource + err := ws.violationBuilder.ResourceRemoval(policy.Name, rkind, rns, rname) + if err != nil { + return err + } + } + } + } + return nil +} diff --git a/pkg/webhooks/registration.go b/pkg/webhooks/registration.go index be5d79a882..b6e63124ad 100644 --- a/pkg/webhooks/registration.go +++ b/pkg/webhooks/registration.go @@ -139,7 +139,8 @@ func (wrc *WebhookRegistrationClient) constructMutatingWebhookConfig(configurati constructWebhook( config.MutatingWebhookName, config.MutatingWebhookServicePath, - caData), + caData, + false), }, }, nil } @@ -157,7 +158,8 @@ func (wrc *WebhookRegistrationClient) contructDebugMutatingWebhookConfig(caData constructDebugWebhook( config.MutatingWebhookName, url, - caData), + caData, + false), }, } } @@ -190,7 +192,8 @@ func (wrc *WebhookRegistrationClient) constructValidatingWebhookConfig(configura constructWebhook( config.ValidatingWebhookName, config.ValidatingWebhookServicePath, - caData), + caData, + true), }, }, nil } @@ -208,7 +211,8 @@ func (wrc *WebhookRegistrationClient) contructDebugValidatingWebhookConfig(caDat constructDebugWebhook( config.ValidatingWebhookName, url, - caData), + caData, + true), }, } } @@ -241,7 +245,8 @@ func (wrc *WebhookRegistrationClient) contructPolicyValidatingWebhookConfig() (* constructWebhook( config.PolicyValidatingWebhookName, config.PolicyValidatingWebhookServicePath, - caData), + caData, + true), }, }, nil } @@ -259,12 +264,13 @@ func (wrc *WebhookRegistrationClient) contructDebugPolicyValidatingWebhookConfig constructDebugWebhook( config.PolicyValidatingWebhookName, url, - caData), + caData, + true), }, } } -func constructWebhook(name, servicePath string, caData []byte) admregapi.Webhook { +func constructWebhook(name, servicePath string, caData []byte, validation bool) admregapi.Webhook { resource := "*/*" apiGroups := "*" apiversions := "*" @@ -273,6 +279,15 @@ func constructWebhook(name, servicePath string, caData []byte) admregapi.Webhook apiGroups = "kyverno.io" apiversions = "v1alpha1" } + operationtypes := []admregapi.OperationType{ + admregapi.Create, + admregapi.Update, + } + // Add operation DELETE for validation + if validation { + operationtypes = append(operationtypes, admregapi.Delete) + + } return admregapi.Webhook{ Name: name, @@ -286,10 +301,7 @@ func constructWebhook(name, servicePath string, caData []byte) admregapi.Webhook }, Rules: []admregapi.RuleWithOperations{ admregapi.RuleWithOperations{ - Operations: []admregapi.OperationType{ - admregapi.Create, - admregapi.Update, - }, + Operations: operationtypes, Rule: admregapi.Rule{ APIGroups: []string{ apiGroups, @@ -306,7 +318,7 @@ func constructWebhook(name, servicePath string, caData []byte) admregapi.Webhook } } -func constructDebugWebhook(name, url string, caData []byte) admregapi.Webhook { +func constructDebugWebhook(name, url string, caData []byte, validation bool) admregapi.Webhook { resource := "*/*" apiGroups := "*" apiversions := "*" @@ -316,6 +328,14 @@ func constructDebugWebhook(name, url string, caData []byte) admregapi.Webhook { apiGroups = "kyverno.io" apiversions = "v1alpha1" } + operationtypes := []admregapi.OperationType{ + admregapi.Create, + admregapi.Update, + } + // Add operation DELETE for validation + if validation { + operationtypes = append(operationtypes, admregapi.Delete) + } return admregapi.Webhook{ Name: name, @@ -325,10 +345,7 @@ func constructDebugWebhook(name, url string, caData []byte) admregapi.Webhook { }, Rules: []admregapi.RuleWithOperations{ admregapi.RuleWithOperations{ - Operations: []admregapi.OperationType{ - admregapi.Create, - admregapi.Update, - }, + Operations: operationtypes, Rule: admregapi.Rule{ APIGroups: []string{ apiGroups, diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index a2b656d558..708754c1b9 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -89,14 +89,30 @@ func (ws *WebhookServer) serve(w http.ResponseWriter, r *http.Request) { // Do not process the admission requests for kinds that are in filterKinds for filtering if !StringInSlice(admissionReview.Request.Kind.Kind, ws.filterKinds) { + // if the resource is being deleted we need to clear any existing Policy Violations + // TODO: can report to the user that we clear the violation corresponding to this resource + if admissionReview.Request.Operation == v1beta1.Delete { + // Resource DELETE + err := ws.removePolicyViolation(admissionReview.Request) + if err != nil { + glog.Info(err) + } + admissionReview.Response = &v1beta1.AdmissionResponse{ + Allowed: true, + } + admissionReview.Response.UID = admissionReview.Request.UID + } else { + // Resource CREATE + // Resource UPDATE + switch r.URL.Path { + case config.MutatingWebhookServicePath: + admissionReview.Response = ws.HandleMutation(admissionReview.Request) + case config.ValidatingWebhookServicePath: + admissionReview.Response = ws.HandleValidation(admissionReview.Request) + case config.PolicyValidatingWebhookServicePath: + admissionReview.Response = ws.HandlePolicyValidation(admissionReview.Request) + } - switch r.URL.Path { - case config.MutatingWebhookServicePath: - admissionReview.Response = ws.HandleMutation(admissionReview.Request) - case config.ValidatingWebhookServicePath: - admissionReview.Response = ws.HandleValidation(admissionReview.Request) - case config.PolicyValidatingWebhookServicePath: - admissionReview.Response = ws.HandlePolicyValidation(admissionReview.Request) } } From a36ed10425f8dd01143a6dc6ff0162516a477c07 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Tue, 16 Jul 2019 15:53:14 -0700 Subject: [PATCH 09/24] change flag & corrections --- pkg/info/info.go | 20 ++++++++++---------- pkg/violation/builder.go | 13 ++++++++++++- pkg/webhooks/mutation.go | 2 +- pkg/webhooks/report.go | 8 ++++++-- pkg/webhooks/utils.go | 13 +++++++++++-- pkg/webhooks/validation.go | 5 +++-- 6 files changed, 43 insertions(+), 18 deletions(-) diff --git a/pkg/info/info.go b/pkg/info/info.go index b19922a4c7..9f0dc004c0 100644 --- a/pkg/info/info.go +++ b/pkg/info/info.go @@ -19,20 +19,20 @@ type PolicyInfo struct { // empty on non-namespaced resources RNamespace string //TODO: add check/enum for types - Mode string // BlockChanges, ReportViolation - Rules []*RuleInfo - success bool + ValidationFailureAction string // BlockChanges, ReportViolation + Rules []*RuleInfo + success bool } //NewPolicyInfo returns a new policy info -func NewPolicyInfo(policyName, rKind, rName, rNamespace, mode string) *PolicyInfo { +func NewPolicyInfo(policyName, rKind, rName, rNamespace, validationFailureAction string) *PolicyInfo { return &PolicyInfo{ - Name: policyName, - RKind: rKind, - RName: rName, - RNamespace: rNamespace, - success: true, // fail to be set explicity - Mode: mode, + Name: policyName, + RKind: rKind, + RName: rName, + RNamespace: rNamespace, + success: true, // fail to be set explicity + ValidationFailureAction: validationFailureAction, } } diff --git a/pkg/violation/builder.go b/pkg/violation/builder.go index 7956ec065b..e15bf88e05 100644 --- a/pkg/violation/builder.go +++ b/pkg/violation/builder.go @@ -82,7 +82,7 @@ func (b *builder) processViolation(info *Info) error { } } // Info: - // Resource - Kind, Namespace, Name + // Key - Kind, Namespace, Name // policy - Name // violation, ok := currVs[info.getKey()] // Key -> resource @@ -232,6 +232,7 @@ func isRuleNamesEqual(currRules []interface{}, newRules []v1alpha1.FailedRule) b return false } glog.Info(reflect.TypeOf(rfule["name"])) + // name name, ok := rfule["name"].(string) if !ok { return false @@ -239,6 +240,16 @@ func isRuleNamesEqual(currRules []interface{}, newRules []v1alpha1.FailedRule) b if name != newRules[i].Name { return false } + // type + + rtype, ok := rfule["type"].(string) + if !ok { + return false + } + if rtype != newRules[i].Type { + return false + } + } return true } diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go index 4038c7d50d..213ae501d6 100644 --- a/pkg/webhooks/mutation.go +++ b/pkg/webhooks/mutation.go @@ -68,7 +68,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be } if len(allPatches) > 0 { - eventsInfo, _ := newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update)) + eventsInfo, _ := newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update), false) ws.eventController.Add(eventsInfo...) } diff --git a/pkg/webhooks/report.go b/pkg/webhooks/report.go index 009fd66462..0573d385df 100644 --- a/pkg/webhooks/report.go +++ b/pkg/webhooks/report.go @@ -10,13 +10,17 @@ import ( "github.com/nirmata/kyverno/pkg/info" ) -func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool) ([]*event.Info, []*violation.Info) { +//TODO: change validation from bool -> enum(validation, mutation) +func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool, validation bool) ([]*event.Info, []*violation.Info) { var eventsInfo []*event.Info var violations []*violation.Info ok, msg := isAdmSuccesful(policyInfoList) // Some policies failed to apply succesfully if !ok { for _, pi := range policyInfoList { + if pi.IsSuccessful() { + continue + } rules := pi.FailedRules() ruleNames := strings.Join(rules, ";") if !onUpdate { @@ -34,7 +38,7 @@ func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool glog.V(3).Infof("Request blocked events info has prepared for %s/%s and %s/%s\n", policyKind, pi.Name, pi.RKind, pi.RName) } // if report flag is set - if pi.Mode == "reportViolation" { + if pi.ValidationFailureAction == ReportViolation && validation { // Create Violations v := violation.BuldNewViolation(pi.Name, pi.RKind, pi.RNamespace, pi.RName, event.PolicyViolation.String(), pi.GetFailedRules()) violations = append(violations, v) diff --git a/pkg/webhooks/utils.go b/pkg/webhooks/utils.go index ce4da62f02..e70fc448c3 100644 --- a/pkg/webhooks/utils.go +++ b/pkg/webhooks/utils.go @@ -85,6 +85,15 @@ func getApplicableKindsForPolicy(p *v1alpha1.Policy) []string { // Policy Reporting Modes const ( - BlockChanges = "blockChanges" - ReportViolation = "reportViolation" + BlockChanges = "block" + ReportViolation = "report" ) + +func toBlock(pis []*info.PolicyInfo) bool { + for _, pi := range pis { + if pi.ValidationFailureAction != ReportViolation { + return true + } + } + return false +} diff --git a/pkg/webhooks/validation.go b/pkg/webhooks/validation.go index 1ecc0af02d..09a5391f20 100644 --- a/pkg/webhooks/validation.go +++ b/pkg/webhooks/validation.go @@ -76,7 +76,7 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 } if len(policyInfos) > 0 && len(policyInfos[0].Rules) != 0 { - eventsInfo, violations = newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update)) + eventsInfo, violations = newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update), true) // If the validationFailureAction flag is set "report", // then we dont block the request and report the violations ws.violationBuilder.Add(violations...) @@ -88,7 +88,8 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 // violations are created if "report" flag is set // and if there are any then we dont bock the resource creation // Even if one the policy being applied - if !ok && violations == nil { + + if !ok && toBlock(policyInfos) { return &v1beta1.AdmissionResponse{ Allowed: false, Result: &metav1.Status{ From bd9e8585c7379c20cce6f13326437c348b3fb6f4 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Wed, 17 Jul 2019 15:04:02 -0700 Subject: [PATCH 10/24] annotations v1 --- pkg/annotations/annotations.go | 220 ++++++++++++++++++++++++++++ pkg/annotations/annotations_test.go | 25 ++++ pkg/dclient/client.go | 7 +- pkg/dclient/utils.go | 215 +++++++++++++++++++++++++++ pkg/kyverno/apply/apply.go | 3 +- pkg/kyverno/apply/util.go | 2 +- pkg/webhooks/mutation.go | 38 ++++- 7 files changed, 506 insertions(+), 4 deletions(-) create mode 100644 pkg/annotations/annotations.go create mode 100644 pkg/annotations/annotations_test.go diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go new file mode 100644 index 0000000000..dd68f1c91e --- /dev/null +++ b/pkg/annotations/annotations.go @@ -0,0 +1,220 @@ +package annotations + +import ( + "encoding/json" + + "github.com/golang/glog" + + "github.com/nirmata/kyverno/pkg/info" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +//Policy information for annotations +type Policy struct { + Status string `json:"status"` + Rules []Rule `json:"rules,omitempty"` +} + +//Rule information for annotations +type Rule struct { + Name string `json:"name"` + Status string `json:"status"` + Type string `json:"type"` + Changes string `json:"changes"` +} + +func getStatus(status bool) string { + if status { + return "Success" + } + return "Failure" +} +func getRules(rules []*info.RuleInfo) []Rule { + var annrules []Rule + for _, r := range rules { + annrule := Rule{Name: r.Name, + Status: getStatus(r.IsSuccessful()), + Type: r.RuleType.String()} + //TODO: add mutation changes in policyInfo and in annotation + annrules = append(annrules, annrule) + } + return annrules +} + +func (p *Policy) updatePolicy(obj *Policy, ruleType info.RuleType) { + p.Status = obj.Status + p.updatePolicyRules(obj.Rules, ruleType) +} + +// Update rules of a given type +func (p *Policy) updatePolicyRules(rules []Rule, ruleType info.RuleType) { + var updatedRules []Rule + //TODO: check the selecting update add advantage + // filter rules for different type + for _, r := range rules { + if r.Type != ruleType.String() { + updatedRules = append(updatedRules, r) + } + } + // Add rules for current type + updatedRules = append(updatedRules, rules...) + // set the rule + p.Rules = updatedRules +} + +// func (p *Policy) containsPolicyRules(rules []Rule, ruleType info.RuleType) { +// for _, r := range rules { +// } +// } +func newAnnotationForPolicy(pi *info.PolicyInfo) *Policy { + return &Policy{Status: getStatus(pi.IsSuccessful()), + Rules: getRules(pi.Rules)} +} + +//AddPolicy will add policy annotation if not present or update if present +func AddPolicy(obj *unstructured.Unstructured, pi *info.PolicyInfo, ruleType info.RuleType) error { + PolicyObj := newAnnotationForPolicy(pi) + // get annotation + ann := obj.GetAnnotations() + // check if policy already has annotation + cPolicy, ok := ann[pi.Name] + if !ok { + PolicyByte, err := json.Marshal(PolicyObj) + if err != nil { + return err + } + // insert policy information + ann[pi.Name] = string(PolicyByte) + // set annotation back to unstr + obj.SetAnnotations(ann) + return nil + } + cPolicyObj := Policy{} + err := json.Unmarshal([]byte(cPolicy), &cPolicyObj) + // update policy information inside the annotation + // 1> policy status + // 2> rule (name, status,changes,type) + cPolicyObj.updatePolicy(PolicyObj, ruleType) + if err != nil { + return err + } + cPolicyByte, err := json.Marshal(cPolicyObj) + if err != nil { + return err + } + // update policy information + ann[pi.Name] = string(cPolicyByte) + // set annotation back to unstr + obj.SetAnnotations(ann) + return nil +} + +//RemovePolicy to remove annotations fro +func RemovePolicy(obj *unstructured.Unstructured, policy string) { + // get annotations + ann := obj.GetAnnotations() + delete(ann, policy) + // set annotation back to unstr + obj.SetAnnotations(ann) +} + +//ParseAnnotationsFromObject extracts annotations from the JSON obj +func ParseAnnotationsFromObject(bytes []byte) map[string]string { + var objectJSON map[string]interface{} + json.Unmarshal(bytes, &objectJSON) + meta, ok := objectJSON["metadata"].(map[string]interface{}) + if !ok { + glog.Error("unable to parse") + return nil + } + if annotations, ok := meta["annotations"].(map[string]string); ok { + return annotations + } + return nil +} + +//AddPolicyJSONPatch generate JSON Patch to add policy informatino JSON patch +func AddPolicyJSONPatch(ann map[string]string, pi *info.PolicyInfo, ruleType info.RuleType) ([]byte, error) { + if ann == nil { + ann = make(map[string]string, 0) + } + PolicyObj := newAnnotationForPolicy(pi) + cPolicy, ok := ann[pi.Name] + if !ok { + PolicyByte, err := json.Marshal(PolicyObj) + if err != nil { + return nil, err + } + // insert policy information + ann[pi.Name] = string(PolicyByte) + // create add JSON patch + return createAddJSONPatch(ann) + } + cPolicyObj := Policy{} + err := json.Unmarshal([]byte(cPolicy), &cPolicyObj) + // update policy information inside the annotation + // 1> policy status + // 2> rule (name, status,changes,type) + cPolicyObj.updatePolicy(PolicyObj, ruleType) + if err != nil { + return nil, err + } + cPolicyByte, err := json.Marshal(cPolicyObj) + if err != nil { + return nil, err + } + // update policy information + ann[pi.Name] = string(cPolicyByte) + // create update JSON patch + return createReplaceJSONPatch(ann) +} + +//RemovePolicyJSONPatch remove JSON patch +func RemovePolicyJSONPatch(ann map[string]string, policy string) ([]byte, error) { + if ann == nil { + return nil, nil + } + delete(ann, policy) + if len(ann) == 0 { + return createRemoveJSONPatch(ann) + } + return createReplaceJSONPatch(ann) +} + +type patchMapValue struct { + Op string `json:"op"` + Path string `json:"path"` + Value map[string]string `json:"value"` +} + +func createRemoveJSONPatch(ann map[string]string) ([]byte, error) { + payload := []patchMapValue{{ + Op: "remove", + Path: "/metadata/annotations", + }} + return json.Marshal(payload) + +} +func createAddJSONPatch(ann map[string]string) ([]byte, error) { + if ann == nil { + ann = make(map[string]string, 0) + } + payload := []patchMapValue{{ + Op: "add", + Path: "/metadata/annotations", + Value: ann, + }} + return json.Marshal(payload) +} + +func createReplaceJSONPatch(ann map[string]string) ([]byte, error) { + if ann == nil { + ann = make(map[string]string, 0) + } + payload := []patchMapValue{{ + Op: "replace", + Path: "/metadata/annotations", + Value: ann, + }} + return json.Marshal(payload) +} diff --git a/pkg/annotations/annotations_test.go b/pkg/annotations/annotations_test.go new file mode 100644 index 0000000000..64dd25c652 --- /dev/null +++ b/pkg/annotations/annotations_test.go @@ -0,0 +1,25 @@ +package annotations + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/nirmata/kyverno/pkg/info" +) + +func TestAddPatch(t *testing.T) { + objRaw := []byte(`{"kind":"Deployment","apiVersion":"apps/v1","metadata":{"name":"nginx-deployment","namespace":"default","creationTimestamp":null,"labels":{"app":"nginx"}},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"nginx"}},"template":{"metadata":{"creationTimestamp":null,"labels":{"app":"nginx"}},"spec":{"containers":[{"name":"nginx","image":"nginx:latest","ports":[{"containerPort":80,"protocol":"TCP"}],"resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"},{"name":"ghost","image":"ghost:latest","resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","securityContext":{},"schedulerName":"default-scheduler"}},"strategy":{"type":"RollingUpdate","rollingUpdate":{"maxUnavailable":"25%","maxSurge":"25%"}},"revisionHistoryLimit":10,"progressDeadlineSeconds":600},"status":{}}`) + piRaw := []byte(`{"Name":"set-image-pull-policy","RKind":"Deployment","RName":"nginx-deployment","RNamespace":"default","ValidationFailureAction":"","Rules":[{"Name":"nginx-deployment","Msgs":["Rule nginx-deployment: Overlay succesfully applied."],"RuleType":0}]}`) + ann := ParseAnnotationsFromObject(objRaw) + pi := info.PolicyInfo{} + err := json.Unmarshal(piRaw, &pi) + if err != nil { + panic(err) + } + patch, err := AddPolicyJSONPatch(ann, &pi, info.Mutation) + if err != nil { + panic(err) + } + fmt.Println(string(patch)) +} diff --git a/pkg/dclient/client.go b/pkg/dclient/client.go index c473221c59..fb591b95b2 100644 --- a/pkg/dclient/client.go +++ b/pkg/dclient/client.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + patchTypes "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/discovery" "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" @@ -40,7 +41,6 @@ func NewClient(config *rest.Config) (*Client, error) { if err != nil { return nil, err } - kclient, err := kubernetes.NewForConfig(config) if err != nil { return nil, err @@ -110,6 +110,11 @@ func (c *Client) GetResource(kind string, namespace string, name string, subreso return c.getResourceInterface(kind, namespace).Get(name, meta.GetOptions{}, subresources...) } +//Patch +func (c *Client) PatchResource(kind string, namespace string, name string, patch []byte) (*unstructured.Unstructured, error) { + return c.getResourceInterface(kind, namespace).Patch(name, patchTypes.JSONPatchType, patch, meta.PatchOptions{}) +} + // ListResource returns the list of resources in unstructured/json format // Access items using []Items func (c *Client) ListResource(kind string, namespace string, lselector *meta.LabelSelector) (*unstructured.UnstructuredList, error) { diff --git a/pkg/dclient/utils.go b/pkg/dclient/utils.go index 6492187038..c982c24574 100644 --- a/pkg/dclient/utils.go +++ b/pkg/dclient/utils.go @@ -1,9 +1,12 @@ package client import ( + "errors" "strings" "time" + "github.com/golang/glog" + "github.com/nirmata/kyverno/pkg/info" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -110,3 +113,215 @@ func retry(attempts int, sleep time.Duration, fn func() error) error { type stop struct { error } + +func GetAnnotations(obj *unstructured.Unstructured) map[string]interface{} { + var annotationsMaps map[string]interface{} + unstr := obj.UnstructuredContent() + metadata, ok := unstr["metadata"] + if ok { + metadataMap, ok := metadata.(map[string]interface{}) + if !ok { + glog.Info("type mismatch") + return nil + } + annotations, ok := metadataMap["annotations"] + if !ok { + glog.Info("annotations not present") + return nil + } + annotationsMaps, ok = annotations.(map[string]interface{}) + if !ok { + glog.Info("type mismatch") + return nil + } + } + return annotationsMaps +} + +func SetAnnotations(obj *unstructured.Unstructured, annotations map[string]interface{}) error { + unstr := obj.UnstructuredContent() + metadata, ok := unstr["metadata"] + if ok { + metadataMap, ok := metadata.(map[string]interface{}) + if !ok { + return errors.New("type mismatch") + } + metadataMap["annotations"] = annotations + unstr["metadata"] = metadataMap + obj.SetUnstructuredContent(unstr) + } + return nil +} + +type AnnotationPolicies struct { + // map[policy_name] + Policies map[string]AnnotationPolicy `json:"policies"` +} + +type AnnotationPolicy struct { + Status string `json:"status"` + Rules []AnnotationRule `json:"rules,omitempty"` +} + +type AnnotationRule struct { + Name string `json:"name"` + Status string `json:"status"` + Type string `json:"type"` + Changes string `json:"changes"` +} + +func getStatus(status bool) string { + if status { + return "Success" + } + return "Failure" +} + +func getRules(rules []*info.RuleInfo) []AnnotationRule { + var annrules []AnnotationRule + for _, r := range rules { + annrule := AnnotationRule{Name: r.Name, + Status: getStatus(r.IsSuccessful())} + //TODO: add mutation changes in policyInfo and in annotation + annrules = append(annrules, annrule) + } + return annrules +} + +// input rules can be mutation or validation +func (ap AnnotationPolicy) updateRules(rules interface{}, validation bool) (error, interface{}) { + ruleList, ok := rules.([]interface{}) + updated := false + if !ok { + return errors.New("type mismatch"), false + } + + // for mutation rule check if the rules are same + // var mode string + // if validation { + // mode = "Validation" + // } else { + // mode = "Mutation" + // } + // // if lengths are differrent then update + // if len(ruleList) != len(ap.Rules) { + // return nil, ap.updateRules + // } + // check if there is any update in the rules + // order of rules is assumed same while comparison + for i, r := range ruleList { + rule, ok := r.(map[string]interface{}) + if !ok { + return errors.New("type mismatch"), nil + } + // Name + name, ok := rule["name"].(string) + if !ok { + return errors.New("type mismatch"), nil + } + if name != ap.Rules[i].Name { + updated = true + break + } + // Status + status, ok := rule["status"].(string) + if !ok { + return errors.New("type mismatch"), nil + } + if status != ap.Rules[i].Status { + updated = true + break + } + } + if updated { + return nil, ap.Rules + } + return nil, nil +} + +func newAnnotationPolicy(pi *info.PolicyInfo) AnnotationPolicy { + status := getStatus(pi.IsSuccessful()) + rules := getRules(pi.Rules) + return AnnotationPolicy{Status: status, + Rules: rules} +} + +//func GetPolicies(policies interface{}) map[string] +func AddPolicy(pi *info.PolicyInfo, ann map[string]interface{}, validation bool) (error, map[string]interface{}) { + // Lets build the policy annotation struct from policyInfo + annpolicy := newAnnotationPolicy(pi) + // Add policy to annotations + // If policy does not exist -> Add + // If already exists then update the status and rules + policies, ok := ann["policies"] + if ok { + policiesMap, ok := policies.(map[string]interface{}) + if !ok { + glog.Info("type mismatch") + return errors.New("type mismatch"), nil + } + // check if policy record is present + policy, ok := policiesMap[pi.Name] + if !ok { + // not present then we add + policiesMap[pi.Name] = annpolicy + ann["policies"] = policiesMap + return nil, ann + } + policyMap, ok := policy.(map[string]interface{}) + if !ok { + return errors.New("type mismatch"), nil + } + // We just update the annotations + // status + status := policyMap["status"] + statusStr, ok := status.(string) + if !ok { + return errors.New("type mismatch"), nil + } + if statusStr != annpolicy.Status { + policyMap["status"] = annpolicy.Status + } + // check rules + rules, ok := policyMap["rules"] + if !ok { + return errors.New("no rules"), nil + } + err, newRules := annpolicy.updateRules(rules, validation) + if err != nil { + return err, nil + } + if newRules == nil { + //nothing to update + return nil, nil + } + // update the new rule + policyMap["rules"] = newRules + // update policies map + policiesMap[pi.Name] = policyMap + ann["policies"] = policiesMap + return nil, ann + } + return nil, nil +} + +// RemovePolicy +func RemovePolicy(pi *info.PolicyInfo, ann map[string]interface{}) (error, map[string]interface{}) { + policies, ok := ann["policies"] + if ok { + policiesMap, ok := policies.(map[string]interface{}) + if !ok { + glog.Info("type mismatch") + return errors.New("type mismatch"), nil + } + // check if policy record is present + _, ok = policiesMap[pi.Name] + if ok { + // delete the pair + delete(policiesMap, pi.Name) + ann["policies"] = policiesMap + return nil, ann + } + } + return nil, nil +} diff --git a/pkg/kyverno/apply/apply.go b/pkg/kyverno/apply/apply.go index baa645609f..eb30c5dc26 100644 --- a/pkg/kyverno/apply/apply.go +++ b/pkg/kyverno/apply/apply.go @@ -103,7 +103,8 @@ func applyPolicyOnRaw(policy *kubepolicy.Policy, rawResource []byte, gvk *metav1 policyInfo := info.NewPolicyInfo(policy.Name, gvk.Kind, rname, - rns) + rns, + policy.Spec.ValidationFailureAction) // Process Mutation patches, ruleInfos := engine.Mutate(*policy, rawResource, *gvk) diff --git a/pkg/kyverno/apply/util.go b/pkg/kyverno/apply/util.go index f46a364961..bb71715ac3 100644 --- a/pkg/kyverno/apply/util.go +++ b/pkg/kyverno/apply/util.go @@ -80,7 +80,7 @@ func scanDir(dir string) ([]string, error) { err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { - return fmt.Errorf("prevent panic by handling failure accessing a path %q: %v\n", dir, err) + return fmt.Errorf("prevent panic by handling failure accessing a path %q: %v", dir, err) } res = append(res, path) diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go index 213ae501d6..d57d18528f 100644 --- a/pkg/webhooks/mutation.go +++ b/pkg/webhooks/mutation.go @@ -1,7 +1,12 @@ package webhooks import ( + "fmt" + + jsonpatch "github.com/evanphx/json-patch" + "github.com/golang/glog" + "github.com/nirmata/kyverno/pkg/annotations" engine "github.com/nirmata/kyverno/pkg/engine" "github.com/nirmata/kyverno/pkg/info" v1beta1 "k8s.io/api/admission/v1beta1" @@ -23,6 +28,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be } var allPatches [][]byte + var annPatches []byte policyInfos := []*info.PolicyInfo{} for _, policy := range policies { // check if policy has a rule for the admission request kind @@ -65,6 +71,18 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be } } policyInfos = append(policyInfos, policyInfo) + annPatch := addAnnotationsToResource(request.Object.Raw, policyInfo, info.Mutation) + if annPatch != nil { + if annPatches == nil { + annPatches = annPatch + } else { + annPatches, err = jsonpatch.MergePatch(annPatches, annPatch) + if err != nil { + fmt.Println("Mergining docs") + fmt.Println(err) + } + } + } } if len(allPatches) > 0 { @@ -74,10 +92,17 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be ok, msg := isAdmSuccesful(policyInfos) if ok { + patches := engine.JoinPatches(allPatches) + if len(annPatches) > 0 { + patches, err = jsonpatch.MergePatch(patches, annPatches) + if err != nil { + fmt.Println(err) + } + } patchType := v1beta1.PatchTypeJSONPatch return &v1beta1.AdmissionResponse{ Allowed: true, - Patch: engine.JoinPatches(allPatches), + Patch: patches, PatchType: &patchType, } } @@ -88,3 +113,14 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be }, } } + +func addAnnotationsToResource(rawResource []byte, pi *info.PolicyInfo, ruleType info.RuleType) []byte { + // get annotations + ann := annotations.ParseAnnotationsFromObject(rawResource) + patch, err := annotations.AddPolicyJSONPatch(ann, pi, ruleType) + if err != nil { + fmt.Println(err) + return nil + } + return patch +} From e5f208e303b37dc5cb22af12282e50430194b4a3 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Wed, 17 Jul 2019 17:53:13 -0700 Subject: [PATCH 11/24] annotation generation from policy controller --- main.go | 11 +- pkg/annotations/annotations.go | 197 ++++++++++++++++++++-------- pkg/annotations/annotations_test.go | 16 ++- pkg/annotations/controller.go | 101 ++++++++++++++ pkg/annotations/utils.go | 20 +++ pkg/controller/controller.go | 89 ++++++++++--- pkg/event/controller.go | 3 +- pkg/webhooks/mutation.go | 13 +- pkg/webhooks/report.go | 12 ++ pkg/webhooks/server.go | 26 ++-- pkg/webhooks/validation.go | 24 +++- 11 files changed, 405 insertions(+), 107 deletions(-) create mode 100644 pkg/annotations/controller.go create mode 100644 pkg/annotations/utils.go diff --git a/main.go b/main.go index a228f43532..116bbb5331 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "flag" "github.com/golang/glog" + "github.com/nirmata/kyverno/pkg/annotations" "github.com/nirmata/kyverno/pkg/config" controller "github.com/nirmata/kyverno/pkg/controller" client "github.com/nirmata/kyverno/pkg/dclient" @@ -24,7 +25,6 @@ var ( func main() { defer glog.Flush() - printVersionInfo() clientConfig, err := createClientConfig(kubeconfig) if err != nil { @@ -43,19 +43,20 @@ func main() { kubeInformer := utils.NewKubeInformerFactory(clientConfig) eventController := event.NewEventController(client, policyInformerFactory) violationBuilder := violation.NewPolicyViolationBuilder(client, policyInformerFactory, eventController) - + annotationsController := annotations.NewAnnotationControler(client) policyController := controller.NewPolicyController( client, policyInformerFactory, violationBuilder, - eventController) + eventController, + annotationsController) genControler := gencontroller.NewGenController(client, eventController, policyInformerFactory, violationBuilder, kubeInformer.Core().V1().Namespaces()) tlsPair, err := initTLSPemPair(clientConfig, client) if err != nil { glog.Fatalf("Failed to initialize TLS key/certificate pair: %v\n", err) } - server, err := webhooks.NewWebhookServer(client, tlsPair, policyInformerFactory, eventController, violationBuilder, filterK8Kinds) + server, err := webhooks.NewWebhookServer(client, tlsPair, policyInformerFactory, eventController, violationBuilder, annotationsController, filterK8Kinds) if err != nil { glog.Fatalf("Unable to create webhook server: %v\n", err) } @@ -71,6 +72,7 @@ func main() { kubeInformer.Start(stopCh) eventController.Run(stopCh) genControler.Run(stopCh) + annotationsController.Run(stopCh) if err = policyController.Run(stopCh); err != nil { glog.Fatalf("Error running PolicyController: %v\n", err) } @@ -84,6 +86,7 @@ func main() { server.Stop() genControler.Stop() eventController.Stop() + annotationsController.Stop() policyController.Stop() } diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index dd68f1c91e..3eb23c3bc3 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -2,24 +2,26 @@ package annotations import ( "encoding/json" + "reflect" "github.com/golang/glog" + pinfo "github.com/nirmata/kyverno/pkg/info" - "github.com/nirmata/kyverno/pkg/info" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) //Policy information for annotations type Policy struct { Status string `json:"status"` - Rules []Rule `json:"rules,omitempty"` + // Key Type/Name + MutationRules map[string]Rule `json:"mutationrules,omitempty"` + ValidationRules map[string]Rule `json:"validationrules,omitempty"` + GenerationRules map[string]Rule `json:"generationrules,omitempty"` } //Rule information for annotations type Rule struct { - Name string `json:"name"` Status string `json:"status"` - Type string `json:"type"` Changes string `json:"changes"` } @@ -29,50 +31,117 @@ func getStatus(status bool) string { } return "Failure" } -func getRules(rules []*info.RuleInfo) []Rule { - var annrules []Rule + +func getRules(rules []*pinfo.RuleInfo, ruleType pinfo.RuleType) map[string]Rule { + if len(rules) == 0 { + return nil + } + annrules := make(map[string]Rule, 0) + // var annrules map[string]Rule for _, r := range rules { - annrule := Rule{Name: r.Name, - Status: getStatus(r.IsSuccessful()), - Type: r.RuleType.String()} - //TODO: add mutation changes in policyInfo and in annotation - annrules = append(annrules, annrule) + if r.RuleType != ruleType { + continue + } + annrules[r.Name] = + Rule{Status: getStatus(r.IsSuccessful())} + // //TODO: add mutation changes in policyInfo and in annotation + // annrules = append(annrules, annrule) } return annrules } -func (p *Policy) updatePolicy(obj *Policy, ruleType info.RuleType) { +func (p *Policy) updatePolicy(obj *Policy, ruleType pinfo.RuleType) bool { + updates := false + if p.Status != obj.Status { + updates = true + } p.Status = obj.Status - p.updatePolicyRules(obj.Rules, ruleType) -} - -// Update rules of a given type -func (p *Policy) updatePolicyRules(rules []Rule, ruleType info.RuleType) { - var updatedRules []Rule - //TODO: check the selecting update add advantage - // filter rules for different type - for _, r := range rules { - if r.Type != ruleType.String() { - updatedRules = append(updatedRules, r) + // Check Mutation rules + switch ruleType { + case pinfo.Mutation: + if p.compareMutationRules(obj.MutationRules) { + updates = true + } + case pinfo.Validation: + if p.compareValidationRules(obj.ValidationRules) { + updates = true + } + case pinfo.Generation: + if p.compareGenerationRules(obj.GenerationRules) { + updates = true } } - // Add rules for current type - updatedRules = append(updatedRules, rules...) - // set the rule - p.Rules = updatedRules + // If there are any updates then the annotation can be updated, can skip + return updates } +func (p *Policy) compareMutationRules(rules map[string]Rule) bool { + // check if the rules have changed + if !reflect.DeepEqual(p.MutationRules, rules) { + p.MutationRules = rules + return true + } + return false +} + +func (p *Policy) compareValidationRules(rules map[string]Rule) bool { + // check if the rules have changed + if !reflect.DeepEqual(p.ValidationRules, rules) { + p.ValidationRules = rules + return true + } + return false +} + +func (p *Policy) compareGenerationRules(rules map[string]Rule) bool { + // check if the rules have changed + if !reflect.DeepEqual(p.GenerationRules, rules) { + p.GenerationRules = rules + return true + } + return false +} + +// // Update rules of a given type +// func (p *Policy) updatePolicyRules(rules map[string]Rule, ruleType info.RuleType) bool { +// updates := false +// // Check if new rules are present in existing rules +// // for k, v := range rules { +// // _, +// // } + +// var updatedRules []Rule +// //TODO: check the selecting update add advantage +// // filter rules for different type +// for _, r := range rules { +// if r.Type != ruleType.String() { +// updatedRules = append(updatedRules, r) +// } +// } +// // Add rules for current type +// updatedRules = append(updatedRules, rules...) +// // set the rule +// p.Rules = updatedRules +// } + // func (p *Policy) containsPolicyRules(rules []Rule, ruleType info.RuleType) { // for _, r := range rules { // } // } -func newAnnotationForPolicy(pi *info.PolicyInfo) *Policy { + +func newAnnotationForPolicy(pi *pinfo.PolicyInfo) *Policy { return &Policy{Status: getStatus(pi.IsSuccessful()), - Rules: getRules(pi.Rules)} + MutationRules: getRules(pi.Rules, pinfo.Mutation), + ValidationRules: getRules(pi.Rules, pinfo.Validation), + GenerationRules: getRules(pi.Rules, pinfo.Generation), + } } //AddPolicy will add policy annotation if not present or update if present -func AddPolicy(obj *unstructured.Unstructured, pi *info.PolicyInfo, ruleType info.RuleType) error { +// modifies obj +// returns true, if there is any update -> caller need to update the obj +// returns false, if there is no change -> caller can skip the update +func AddPolicy(obj *unstructured.Unstructured, pi *pinfo.PolicyInfo, ruleType pinfo.RuleType) bool { PolicyObj := newAnnotationForPolicy(pi) // get annotation ann := obj.GetAnnotations() @@ -81,41 +150,50 @@ func AddPolicy(obj *unstructured.Unstructured, pi *info.PolicyInfo, ruleType inf if !ok { PolicyByte, err := json.Marshal(PolicyObj) if err != nil { - return err + glog.Error(err) + return false } // insert policy information ann[pi.Name] = string(PolicyByte) // set annotation back to unstr obj.SetAnnotations(ann) - return nil + return true } cPolicyObj := Policy{} err := json.Unmarshal([]byte(cPolicy), &cPolicyObj) + if err != nil { + return false + } // update policy information inside the annotation // 1> policy status - // 2> rule (name, status,changes,type) - cPolicyObj.updatePolicy(PolicyObj, ruleType) - if err != nil { - return err + // 2> Mutation, Validation, Generation + if cPolicyObj.updatePolicy(PolicyObj, ruleType) { + cPolicyByte, err := json.Marshal(cPolicyObj) + if err != nil { + return false + } + // update policy information + ann[pi.Name] = string(cPolicyByte) + // set annotation back to unstr + obj.SetAnnotations(ann) + return true } - cPolicyByte, err := json.Marshal(cPolicyObj) - if err != nil { - return err - } - // update policy information - ann[pi.Name] = string(cPolicyByte) - // set annotation back to unstr - obj.SetAnnotations(ann) - return nil + return false } -//RemovePolicy to remove annotations fro -func RemovePolicy(obj *unstructured.Unstructured, policy string) { +//RemovePolicy to remove annotations +// return true -> if there was an entry and we deleted it +// return false -> if there is no entry, caller need not update +func RemovePolicy(obj *unstructured.Unstructured, policy string) bool { // get annotations ann := obj.GetAnnotations() + if _, ok := ann[policy]; !ok { + return false + } delete(ann, policy) // set annotation back to unstr obj.SetAnnotations(ann) + return true } //ParseAnnotationsFromObject extracts annotations from the JSON obj @@ -134,7 +212,7 @@ func ParseAnnotationsFromObject(bytes []byte) map[string]string { } //AddPolicyJSONPatch generate JSON Patch to add policy informatino JSON patch -func AddPolicyJSONPatch(ann map[string]string, pi *info.PolicyInfo, ruleType info.RuleType) ([]byte, error) { +func AddPolicyJSONPatch(ann map[string]string, pi *pinfo.PolicyInfo, ruleType pinfo.RuleType) (map[string]string, []byte, error) { if ann == nil { ann = make(map[string]string, 0) } @@ -143,12 +221,13 @@ func AddPolicyJSONPatch(ann map[string]string, pi *info.PolicyInfo, ruleType inf if !ok { PolicyByte, err := json.Marshal(PolicyObj) if err != nil { - return nil, err + return nil, nil, err } // insert policy information ann[pi.Name] = string(PolicyByte) // create add JSON patch - return createAddJSONPatch(ann) + jsonPatch, err := createAddJSONPatch(ann) + return ann, jsonPatch, err } cPolicyObj := Policy{} err := json.Unmarshal([]byte(cPolicy), &cPolicyObj) @@ -157,28 +236,31 @@ func AddPolicyJSONPatch(ann map[string]string, pi *info.PolicyInfo, ruleType inf // 2> rule (name, status,changes,type) cPolicyObj.updatePolicy(PolicyObj, ruleType) if err != nil { - return nil, err + return nil, nil, err } cPolicyByte, err := json.Marshal(cPolicyObj) if err != nil { - return nil, err + return nil, nil, err } // update policy information ann[pi.Name] = string(cPolicyByte) // create update JSON patch - return createReplaceJSONPatch(ann) + jsonPatch, err := createReplaceJSONPatch(ann) + return ann, jsonPatch, err } //RemovePolicyJSONPatch remove JSON patch -func RemovePolicyJSONPatch(ann map[string]string, policy string) ([]byte, error) { +func RemovePolicyJSONPatch(ann map[string]string, policy string) (map[string]string, []byte, error) { if ann == nil { - return nil, nil + return nil, nil, nil } delete(ann, policy) if len(ann) == 0 { - return createRemoveJSONPatch(ann) + jsonPatch, err := createRemoveJSONPatch(ann) + return nil, jsonPatch, err } - return createReplaceJSONPatch(ann) + jsonPatch, err := createReplaceJSONPatch(ann) + return ann, jsonPatch, err } type patchMapValue struct { @@ -195,6 +277,7 @@ func createRemoveJSONPatch(ann map[string]string) ([]byte, error) { return json.Marshal(payload) } + func createAddJSONPatch(ann map[string]string) ([]byte, error) { if ann == nil { ann = make(map[string]string, 0) diff --git a/pkg/annotations/annotations_test.go b/pkg/annotations/annotations_test.go index 64dd25c652..58741ed824 100644 --- a/pkg/annotations/annotations_test.go +++ b/pkg/annotations/annotations_test.go @@ -9,6 +9,7 @@ import ( ) func TestAddPatch(t *testing.T) { + // Create objRaw := []byte(`{"kind":"Deployment","apiVersion":"apps/v1","metadata":{"name":"nginx-deployment","namespace":"default","creationTimestamp":null,"labels":{"app":"nginx"}},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"nginx"}},"template":{"metadata":{"creationTimestamp":null,"labels":{"app":"nginx"}},"spec":{"containers":[{"name":"nginx","image":"nginx:latest","ports":[{"containerPort":80,"protocol":"TCP"}],"resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"},{"name":"ghost","image":"ghost:latest","resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","securityContext":{},"schedulerName":"default-scheduler"}},"strategy":{"type":"RollingUpdate","rollingUpdate":{"maxUnavailable":"25%","maxSurge":"25%"}},"revisionHistoryLimit":10,"progressDeadlineSeconds":600},"status":{}}`) piRaw := []byte(`{"Name":"set-image-pull-policy","RKind":"Deployment","RName":"nginx-deployment","RNamespace":"default","ValidationFailureAction":"","Rules":[{"Name":"nginx-deployment","Msgs":["Rule nginx-deployment: Overlay succesfully applied."],"RuleType":0}]}`) ann := ParseAnnotationsFromObject(objRaw) @@ -17,7 +18,20 @@ func TestAddPatch(t *testing.T) { if err != nil { panic(err) } - patch, err := AddPolicyJSONPatch(ann, &pi, info.Mutation) + ann, patch, err := AddPolicyJSONPatch(ann, &pi, info.Mutation) + if err != nil { + panic(err) + } + fmt.Println(string(patch)) + // Update + piRaw = []byte(`{"Name":"set-image-pull-policy","RKind":"Deployment","RName":"nginx-deployment","RNamespace":"default","ValidationFailureAction":"","Rules":[{"Name":"nginx-deployment","Msgs":["Rule nginx-deployment1: Overlay succesfully applied."],"RuleType":0}]}`) + // ann = ParseAnnotationsFromObject(objRaw) + pi = info.PolicyInfo{} + err = json.Unmarshal(piRaw, &pi) + if err != nil { + panic(err) + } + ann, patch, err = AddPolicyJSONPatch(ann, &pi, info.Mutation) if err != nil { panic(err) } diff --git a/pkg/annotations/controller.go b/pkg/annotations/controller.go new file mode 100644 index 0000000000..a92b7279c3 --- /dev/null +++ b/pkg/annotations/controller.go @@ -0,0 +1,101 @@ +package annotations + +import ( + "fmt" + "time" + + "github.com/golang/glog" + client "github.com/nirmata/kyverno/pkg/dclient" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/workqueue" +) + +type controller struct { + client *client.Client + queue workqueue.RateLimitingInterface +} + +type Interface interface { + Add(rkind, rns, rname string, patch []byte) +} + +type Controller interface { + Interface + Run(stopCh <-chan struct{}) + Stop() +} + +func NewAnnotationControler(client *client.Client) Controller { + return &controller{ + client: client, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), annotationQueueName), + } +} + +func (c *controller) Add(rkind, rns, rname string, patch []byte) { + c.queue.Add(newInfo) +} + +func (c *controller) Run(stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + for i := 0; i < workerThreadCount; i++ { + go wait.Until(c.runWorker, time.Second, stopCh) + } + glog.Info("Started annotation controller workers") +} + +func (c *controller) Stop() { + defer c.queue.ShutDown() + glog.Info("Shutting down annotation controller workers") +} + +func (c *controller) runWorker() { + for c.processNextWorkItem() { + } +} + +func (c *controller) processNextWorkItem() bool { + obj, shutdown := c.queue.Get() + if shutdown { + return false + } + err := func(obj interface{}) error { + defer c.queue.Done(obj) + var key info + var ok bool + if key, ok = obj.(info); !ok { + c.queue.Forget(obj) + glog.Warningf("Expecting type info by got %v\n", obj) + return nil + } + // Run the syncHandler, passing the resource and the policy + if err := c.SyncHandler(key); err != nil { + c.queue.AddRateLimited(key) + return fmt.Errorf("error syncing '%s/%s/%s' : %s, requeuing annotation creation request", key.RKind, key.RNs, key.RName, err) + } + return nil + }(obj) + + if err != nil { + glog.Warning(err) + } + return true +} + +func (c *controller) SyncHandler(key info) error { + var err error + // check if the resource is created + _, err = c.client.GetResource(key.RKind, key.RNs, key.RName) + if err != nil { + glog.Errorf("Error creating annotation: unable to get resource %s/%s/%s, will retry: %s ", key.RKind, key.RNs, key.RName, err) + return err + } + // if it is patch the resource + _, err = c.client.PatchResource(key.RKind, key.RNs, key.RName, key.Patch) + if err != nil { + glog.Errorf("Error creating annotation: unable to get resource %s/%s/%s, will retry: %s", key.RKind, key.RNs, key.RName, err) + return err + } + return nil +} diff --git a/pkg/annotations/utils.go b/pkg/annotations/utils.go new file mode 100644 index 0000000000..ae2e8ab53e --- /dev/null +++ b/pkg/annotations/utils.go @@ -0,0 +1,20 @@ +package annotations + +const annotationQueueName = "annotation-queue" +const workerThreadCount = 1 + +type info struct { + RKind string + RNs string + RName string + Patch []byte +} + +func newInfo(rkind, rns, rname string, patch []byte) info { + return info{ + RKind: rkind, + RNs: rname, + RName: rname, + Patch: patch, + } +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 524a144a1b..f299f99658 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -5,6 +5,9 @@ import ( "reflect" "time" + jsonpatch "github.com/evanphx/json-patch" + + "github.com/nirmata/kyverno/pkg/annotations" "github.com/nirmata/kyverno/pkg/info" "github.com/nirmata/kyverno/pkg/engine" @@ -26,27 +29,30 @@ import ( //PolicyController to manage Policy CRD type PolicyController struct { - client *client.Client - policyLister lister.PolicyLister - policySynced cache.InformerSynced - violationBuilder violation.Generator - eventController event.Generator - queue workqueue.RateLimitingInterface + client *client.Client + policyLister lister.PolicyLister + policySynced cache.InformerSynced + violationBuilder violation.Generator + eventController event.Generator + annotationsController annotations.Controller + queue workqueue.RateLimitingInterface } // NewPolicyController from cmd args func NewPolicyController(client *client.Client, policyInformer sharedinformer.PolicyInformer, violationBuilder violation.Generator, - eventController event.Generator) *PolicyController { + eventController event.Generator, + annotationsController annotations.Controller) *PolicyController { controller := &PolicyController{ - client: client, - policyLister: policyInformer.GetLister(), - policySynced: policyInformer.GetInfomer().HasSynced, - violationBuilder: violationBuilder, - eventController: eventController, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), policyWorkQueueName), + client: client, + policyLister: policyInformer.GetLister(), + policySynced: policyInformer.GetInfomer().HasSynced, + violationBuilder: violationBuilder, + eventController: eventController, + annotationsController: annotationsController, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), policyWorkQueueName), } policyInformer.GetInfomer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -81,6 +87,7 @@ func (pc *PolicyController) deletePolicyHandler(resource interface{}) { glog.Error("error decoding object, invalid type") return } + //TODO: need to clear annotations on the resources glog.Infof("policy deleted: %s", object.GetName()) } @@ -190,9 +197,62 @@ func (pc *PolicyController) syncHandler(obj interface{}) error { if err != nil { glog.Error(err) } + + // add annotations to resources + pc.createAnnotations(policyInfos) + return nil } +func (pc *PolicyController) createAnnotations(policyInfos []*info.PolicyInfo) { + for _, pi := range policyInfos { + var patch []byte + //get resource + obj, err := pc.client.GetResource(pi.RKind, pi.RNamespace, pi.RName) + if err != nil { + glog.Error(err) + continue + } + // add annotation for policy application + ann := obj.GetAnnotations() + // Mutation rules + ann, mpatch, err := annotations.AddPolicyJSONPatch(ann, pi, info.Mutation) + if err != nil { + glog.Error(err) + continue + } + // Validation rules + ann, vpatch, err := annotations.AddPolicyJSONPatch(ann, pi, info.Validation) + if err != nil { + glog.Error(err) + } + if mpatch == nil && mpatch == nil { + //nothing to patch + continue + } + // merge the patches + if mpatch != nil && vpatch != nil { + patch, err = jsonpatch.MergePatch(mpatch, vpatch) + if err != nil { + glog.Error(err) + continue + } + } + if mpatch == nil { + patch = vpatch + } else { + patch = vpatch + } + + // add the anotation to the resource + _, err = pc.client.PatchResource(pi.RKind, pi.RNamespace, pi.RName, patch) + if err != nil { + glog.Error(err) + continue + } + } +} + func (pc *PolicyController) createEventsAndViolations(policyInfos []*info.PolicyInfo) ([]*event.Info, []*violation.Info) { events := []*event.Info{} violations := []*violation.Info{} @@ -228,9 +288,6 @@ func (pc *PolicyController) createEventsAndViolations(policyInfos []*info.Policy } if !policyInfo.IsSuccessful() { - // Event - // list of failed rules : ruleNames - e := event.NewEvent("Policy", "", policyInfo.Name, event.PolicyViolation, event.FResourcePolcy, policyInfo.RNamespace+"/"+policyInfo.RName, concatFailedRules(frules)) events = append(events, e) // Violation diff --git a/pkg/event/controller.go b/pkg/event/controller.go index e519238f67..819f97cf73 100644 --- a/pkg/event/controller.go +++ b/pkg/event/controller.go @@ -42,13 +42,12 @@ type Controller interface { func NewEventController(client *client.Client, shareInformer sharedinformer.PolicyInformer) Controller { - controller := &controller{ + return &controller{ client: client, policyLister: shareInformer.GetLister(), queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), eventWorkQueueName), recorder: initRecorder(client), } - return controller } func initRecorder(client *client.Client) record.EventRecorder { diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go index d57d18528f..aa762dab07 100644 --- a/pkg/webhooks/mutation.go +++ b/pkg/webhooks/mutation.go @@ -6,7 +6,6 @@ import ( jsonpatch "github.com/evanphx/json-patch" "github.com/golang/glog" - "github.com/nirmata/kyverno/pkg/annotations" engine "github.com/nirmata/kyverno/pkg/engine" "github.com/nirmata/kyverno/pkg/info" v1beta1 "k8s.io/api/admission/v1beta1" @@ -78,6 +77,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be } else { annPatches, err = jsonpatch.MergePatch(annPatches, annPatch) if err != nil { + glog.Error(err) fmt.Println("Mergining docs") fmt.Println(err) } @@ -113,14 +113,3 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be }, } } - -func addAnnotationsToResource(rawResource []byte, pi *info.PolicyInfo, ruleType info.RuleType) []byte { - // get annotations - ann := annotations.ParseAnnotationsFromObject(rawResource) - patch, err := annotations.AddPolicyJSONPatch(ann, pi, ruleType) - if err != nil { - fmt.Println(err) - return nil - } - return patch -} diff --git a/pkg/webhooks/report.go b/pkg/webhooks/report.go index 0573d385df..4850cd4a77 100644 --- a/pkg/webhooks/report.go +++ b/pkg/webhooks/report.go @@ -3,6 +3,7 @@ package webhooks import ( "strings" + "github.com/nirmata/kyverno/pkg/annotations" "github.com/nirmata/kyverno/pkg/violation" "github.com/golang/glog" @@ -60,3 +61,14 @@ func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool } return eventsInfo, violations } + +func addAnnotationsToResource(rawResource []byte, pi *info.PolicyInfo, ruleType info.RuleType) []byte { + // get annotations + ann := annotations.ParseAnnotationsFromObject(rawResource) + ann, patch, err := annotations.AddPolicyJSONPatch(ann, pi, ruleType) + if err != nil { + glog.Error(err) + return nil + } + return patch +} diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index 708754c1b9..515b1fb4d7 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -11,6 +11,7 @@ import ( "time" "github.com/golang/glog" + "github.com/nirmata/kyverno/pkg/annotations" "github.com/nirmata/kyverno/pkg/client/listers/policy/v1alpha1" "github.com/nirmata/kyverno/pkg/config" client "github.com/nirmata/kyverno/pkg/dclient" @@ -24,12 +25,13 @@ import ( // WebhookServer contains configured TLS server with MutationWebhook. // MutationWebhook gets policies from policyController and takes control of the cluster with kubeclient. type WebhookServer struct { - server http.Server - client *client.Client - policyLister v1alpha1.PolicyLister - eventController event.Generator - violationBuilder violation.Generator - filterKinds []string + server http.Server + client *client.Client + policyLister v1alpha1.PolicyLister + eventController event.Generator + violationBuilder violation.Generator + annotationsController annotations.Controller + filterKinds []string } // NewWebhookServer creates new instance of WebhookServer accordingly to given configuration @@ -40,6 +42,7 @@ func NewWebhookServer( shareInformer sharedinformer.PolicyInformer, eventController event.Generator, violationBuilder violation.Generator, + annotationsController annotations.Controller, filterKinds []string) (*WebhookServer, error) { if tlsPair == nil { @@ -54,11 +57,12 @@ func NewWebhookServer( tlsConfig.Certificates = []tls.Certificate{pair} ws := &WebhookServer{ - client: client, - policyLister: shareInformer.GetLister(), - eventController: eventController, - violationBuilder: violationBuilder, - filterKinds: parseKinds(filterKinds), + client: client, + policyLister: shareInformer.GetLister(), + eventController: eventController, + violationBuilder: violationBuilder, + annotationsController: annotationsController, + filterKinds: parseKinds(filterKinds), } mux := http.NewServeMux() mux.HandleFunc(config.MutatingWebhookServicePath, ws.serve) diff --git a/pkg/webhooks/validation.go b/pkg/webhooks/validation.go index 09a5391f20..7ee7f20363 100644 --- a/pkg/webhooks/validation.go +++ b/pkg/webhooks/validation.go @@ -1,6 +1,7 @@ package webhooks import ( + jsonpatch "github.com/evanphx/json-patch" "github.com/golang/glog" engine "github.com/nirmata/kyverno/pkg/engine" "github.com/nirmata/kyverno/pkg/event" @@ -26,15 +27,16 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 Allowed: true, } } + rname := engine.ParseNameFromObject(request.Object.Raw) + rns := engine.ParseNamespaceFromObject(request.Object.Raw) + rkind := engine.ParseKindFromObject(request.Object.Raw) + var annPatches []byte for _, policy := range policies { if !StringInSlice(request.Kind.Kind, getApplicableKindsForPolicy(policy)) { continue } - rname := engine.ParseNameFromObject(request.Object.Raw) - rns := engine.ParseNamespaceFromObject(request.Object.Raw) - rkind := engine.ParseKindFromObject(request.Object.Raw) policyInfo := info.NewPolicyInfo(policy.Name, rkind, @@ -73,6 +75,17 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 } } policyInfos = append(policyInfos, policyInfo) + annPatch := addAnnotationsToResource(request.Object.Raw, policyInfo, info.Mutation) + if annPatch != nil { + if annPatches == nil { + annPatches = annPatch + } else { + annPatches, err = jsonpatch.MergePatch(annPatches, annPatch) + if err != nil { + glog.Error(err) + } + } + } } if len(policyInfos) > 0 && len(policyInfos[0].Rules) != 0 { @@ -82,7 +95,10 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 ws.violationBuilder.Add(violations...) ws.eventController.Add(eventsInfo...) } - + // add annotations + if annPatches != nil { + ws.annotationsController.Add(rkind, rns, rname, annPatches) + } // If Validation fails then reject the request ok, msg := isAdmSuccesful(policyInfos) // violations are created if "report" flag is set From 129ced1b2aedb9a18d0bcb926e2244218c0e5c44 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Wed, 17 Jul 2019 23:13:28 -0700 Subject: [PATCH 12/24] annotations creation,update & removal --- pkg/annotations/annotations.go | 22 +++++++--- pkg/annotations/controller.go | 54 ++++++++++++++--------- pkg/annotations/utils.go | 8 ++-- pkg/controller/cleanup.go | 78 ++++++++++++++++++++++++++++++++++ pkg/controller/controller.go | 16 +++---- pkg/engine/engine.go | 14 +++--- pkg/webhooks/deleteresource.go | 2 +- pkg/webhooks/mutation.go | 14 +++--- pkg/webhooks/report.go | 5 +++ pkg/webhooks/validation.go | 22 +++++----- 10 files changed, 171 insertions(+), 64 deletions(-) create mode 100644 pkg/controller/cleanup.go diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 3eb23c3bc3..360debe536 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -22,7 +22,7 @@ type Policy struct { //Rule information for annotations type Rule struct { Status string `json:"status"` - Changes string `json:"changes"` + Changes string `json:"changes,omitempty"` // TODO for mutation changes } func getStatus(status bool) string { @@ -187,6 +187,9 @@ func AddPolicy(obj *unstructured.Unstructured, pi *pinfo.PolicyInfo, ruleType pi func RemovePolicy(obj *unstructured.Unstructured, policy string) bool { // get annotations ann := obj.GetAnnotations() + if ann == nil { + return false + } if _, ok := ann[policy]; !ok { return false } @@ -205,10 +208,11 @@ func ParseAnnotationsFromObject(bytes []byte) map[string]string { glog.Error("unable to parse") return nil } - if annotations, ok := meta["annotations"].(map[string]string); ok { - return annotations + ann, ok, err := unstructured.NestedStringMap(meta, "annotations") + if err != nil || !ok { + return nil } - return nil + return ann } //AddPolicyJSONPatch generate JSON Patch to add policy informatino JSON patch @@ -227,6 +231,12 @@ func AddPolicyJSONPatch(ann map[string]string, pi *pinfo.PolicyInfo, ruleType pi ann[pi.Name] = string(PolicyByte) // create add JSON patch jsonPatch, err := createAddJSONPatch(ann) + // var jsonPatch []byte + // if len(ann) == 0 { + // jsonPatch, err = createAddJSONPatch(ann) + // } else { + // jsonPatch, err = createReplaceJSONPatch(ann) + // } return ann, jsonPatch, err } cPolicyObj := Policy{} @@ -234,8 +244,8 @@ func AddPolicyJSONPatch(ann map[string]string, pi *pinfo.PolicyInfo, ruleType pi // update policy information inside the annotation // 1> policy status // 2> rule (name, status,changes,type) - cPolicyObj.updatePolicy(PolicyObj, ruleType) - if err != nil { + update := cPolicyObj.updatePolicy(PolicyObj, ruleType) + if !update { return nil, nil, err } cPolicyByte, err := json.Marshal(cPolicyObj) diff --git a/pkg/annotations/controller.go b/pkg/annotations/controller.go index a92b7279c3..cfdb5ec12a 100644 --- a/pkg/annotations/controller.go +++ b/pkg/annotations/controller.go @@ -34,7 +34,7 @@ func NewAnnotationControler(client *client.Client) Controller { } func (c *controller) Add(rkind, rns, rname string, patch []byte) { - c.queue.Add(newInfo) + c.queue.Add(newInfo(rkind, rns, rname, patch)) } func (c *controller) Run(stopCh <-chan struct{}) { @@ -55,35 +55,49 @@ func (c *controller) runWorker() { } } -func (c *controller) processNextWorkItem() bool { - obj, shutdown := c.queue.Get() +func (pc *controller) processNextWorkItem() bool { + obj, shutdown := pc.queue.Get() if shutdown { return false } + err := func(obj interface{}) error { - defer c.queue.Done(obj) - var key info - var ok bool - if key, ok = obj.(info); !ok { - c.queue.Forget(obj) - glog.Warningf("Expecting type info by got %v\n", obj) - return nil - } - // Run the syncHandler, passing the resource and the policy - if err := c.SyncHandler(key); err != nil { - c.queue.AddRateLimited(key) - return fmt.Errorf("error syncing '%s/%s/%s' : %s, requeuing annotation creation request", key.RKind, key.RNs, key.RName, err) - } + defer pc.queue.Done(obj) + err := pc.syncHandler(obj) + pc.handleErr(err, obj) return nil }(obj) - if err != nil { - glog.Warning(err) + glog.Error(err) + return true } return true } +func (pc *controller) handleErr(err error, key interface{}) { + if err == nil { + pc.queue.Forget(key) + return + } + // This controller retries if something goes wrong. After that, it stops trying. + if pc.queue.NumRequeues(key) < WorkQueueRetryLimit { + glog.Warningf("Error syncing events %v: %v", key, err) + // Re-enqueue the key rate limited. Based on the rate limiter on the + // queue and the re-enqueue history, the key will be processed later again. + pc.queue.AddRateLimited(key) + return + } + pc.queue.Forget(key) + glog.Error(err) + glog.Warningf("Dropping the key %q out of the queue: %v", key, err) +} + +func (c *controller) syncHandler(obj interface{}) error { + var key info + var ok bool + if key, ok = obj.(info); !ok { + return fmt.Errorf("expected string in workqueue but got %#v", obj) + } -func (c *controller) SyncHandler(key info) error { var err error // check if the resource is created _, err = c.client.GetResource(key.RKind, key.RNs, key.RName) @@ -92,7 +106,7 @@ func (c *controller) SyncHandler(key info) error { return err } // if it is patch the resource - _, err = c.client.PatchResource(key.RKind, key.RNs, key.RName, key.Patch) + _, err = c.client.PatchResource(key.RKind, key.RNs, key.RName, *key.Patch) if err != nil { glog.Errorf("Error creating annotation: unable to get resource %s/%s/%s, will retry: %s", key.RKind, key.RNs, key.RName, err) return err diff --git a/pkg/annotations/utils.go b/pkg/annotations/utils.go index ae2e8ab53e..7ee8e69ba4 100644 --- a/pkg/annotations/utils.go +++ b/pkg/annotations/utils.go @@ -2,19 +2,21 @@ package annotations const annotationQueueName = "annotation-queue" const workerThreadCount = 1 +const WorkQueueRetryLimit = 3 type info struct { RKind string RNs string RName string - Patch []byte + //TODO:Hack as slice makes the struct unhasable + Patch *[]byte } func newInfo(rkind, rns, rname string, patch []byte) info { return info{ RKind: rkind, - RNs: rname, + RNs: rns, RName: rname, - Patch: patch, + Patch: &patch, } } diff --git a/pkg/controller/cleanup.go b/pkg/controller/cleanup.go new file mode 100644 index 0000000000..486bcdc783 --- /dev/null +++ b/pkg/controller/cleanup.go @@ -0,0 +1,78 @@ +package controller + +import ( + "github.com/golang/glog" + "github.com/minio/minio/pkg/wildcard" + "github.com/nirmata/kyverno/pkg/annotations" + v1alpha1 "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" + client "github.com/nirmata/kyverno/pkg/dclient" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" +) + +func cleanAnnotations(client *client.Client, obj interface{}) { + // get the policy struct from interface + unstr, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + glog.Error(err) + return + } + policy := v1alpha1.Policy{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstr, &policy); err != nil { + glog.Error(err) + return + } + // Get the resources that apply to the policy + // key uid + resourceMap := map[string]unstructured.Unstructured{} + for _, rule := range policy.Spec.Rules { + for _, k := range rule.Kinds { + if k == "Namespace" { + // REWORK: will be handeled by namespace controller + continue + } + // kind -> resource + gvr := client.DiscoveryClient.GetGVRFromKind(k) + // label selectors + // namespace ? should it be default or allow policy to specify it + namespace := "default" + if rule.ResourceDescription.Namespace != nil { + namespace = *rule.ResourceDescription.Namespace + } + list, err := client.ListResource(k, namespace, rule.ResourceDescription.Selector) + if err != nil { + glog.Errorf("unable to list resource for %s with label selector %s", gvr.Resource, rule.Selector.String()) + glog.Errorf("unable to apply policy %s rule %s. err: %s", policy.Name, rule.Name, err) + continue + } + for _, res := range list.Items { + name := rule.ResourceDescription.Name + if name != nil { + // wild card matching + if !wildcard.Match(*name, res.GetName()) { + continue + } + } + resourceMap[string(res.GetUID())] = res + } + } + } + + // remove annotations for the resources + for _, obj := range resourceMap { + // get annotations + ann := obj.GetAnnotations() + + _, patch, err := annotations.RemovePolicyJSONPatch(ann, policy.Name) + if err != nil { + glog.Error(err) + continue + } + // patch the resource + _, err = client.PatchResource(obj.GetKind(), obj.GetNamespace(), obj.GetName(), patch) + if err != nil { + glog.Error(err) + continue + } + } +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index f299f99658..bc14db812c 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -88,6 +88,7 @@ func (pc *PolicyController) deletePolicyHandler(resource interface{}) { return } //TODO: need to clear annotations on the resources + cleanAnnotations(pc.client, resource) glog.Infof("policy deleted: %s", object.GetName()) } @@ -191,12 +192,12 @@ func (pc *PolicyController) syncHandler(obj interface{}) error { //TODO: processPolicy glog.Infof("process policy %s on existing resources", policy.GetName()) policyInfos := engine.ProcessExisting(pc.client, policy) - events, violations := pc.createEventsAndViolations(policyInfos) + events, _ := pc.createEventsAndViolations(policyInfos) pc.eventController.Add(events...) - err = pc.violationBuilder.Add(violations...) - if err != nil { - glog.Error(err) - } + // err = pc.violationBuilder.Add(violations...) + // if err != nil { + // glog.Error(err) + // } // add annotations to resources pc.createAnnotations(policyInfos) @@ -241,10 +242,9 @@ func (pc *PolicyController) createAnnotations(policyInfos []*info.PolicyInfo) { if mpatch == nil { patch = vpatch } else { - patch = vpatch + patch = mpatch } - - // add the anotation to the resource + // add the anotation to the resource _, err = pc.client.PatchResource(pi.RKind, pi.RNamespace, pi.RName, patch) if err != nil { glog.Error(err) diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 29afad4375..79142d9269 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -14,11 +14,11 @@ import ( // Instead we expose them as standalone functions passing the required atrributes // The each function returns the changes that need to be applied on the resource // the caller is responsible to apply the changes to the resource - // ProcessExisting checks for mutation and validation violations of existing resources func ProcessExisting(client *client.Client, policy *types.Policy) []*info.PolicyInfo { glog.Infof("Applying policy %s on existing resources", policy.Name) - resources := []*resourceInfo{} + // key uid + resourceMap := map[string]*resourceInfo{} for _, rule := range policy.Spec.Rules { for _, k := range rule.Kinds { @@ -52,18 +52,20 @@ func ProcessExisting(client *client.Client, policy *types.Policy) []*info.Policy ri := &resourceInfo{resource: &res, gvk: &metav1.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind}} - resources = append(resources, ri) + // resources = append(resources, ri) + + resourceMap[string(res.GetUID())] = ri } } } policyInfos := []*info.PolicyInfo{} // for the filtered resource apply policy - for _, r := range resources { + for _, v := range resourceMap { - policyInfo, err := applyPolicy(client, policy, r) + policyInfo, err := applyPolicy(client, policy, v) if err != nil { - glog.Errorf("unable to apply policy %s on resource %s/%s", policy.Name, r.resource.GetName(), r.resource.GetNamespace()) + glog.Errorf("unable to apply policy %s on resource %s/%s", policy.Name, v.resource.GetName(), v.resource.GetNamespace()) glog.Error(err) continue } diff --git a/pkg/webhooks/deleteresource.go b/pkg/webhooks/deleteresource.go index 7d5743f33b..a1449791db 100644 --- a/pkg/webhooks/deleteresource.go +++ b/pkg/webhooks/deleteresource.go @@ -9,7 +9,7 @@ import ( ) func (ws *WebhookServer) removePolicyViolation(request *v1beta1.AdmissionRequest) error { - + return nil // Get the list of policies that apply on the resource policies, err := ws.policyLister.List(labels.NewSelector()) if err != nil { diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go index aa762dab07..957b5abf79 100644 --- a/pkg/webhooks/mutation.go +++ b/pkg/webhooks/mutation.go @@ -1,8 +1,6 @@ package webhooks import ( - "fmt" - jsonpatch "github.com/evanphx/json-patch" "github.com/golang/glog" @@ -59,10 +57,10 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be } } else { // CleanUp Violations if exists - err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Mutation) - if err != nil { - glog.Info(err) - } + // err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Mutation) + // if err != nil { + // glog.Info(err) + // } if len(policyPatches) > 0 { allPatches = append(allPatches, policyPatches...) @@ -78,8 +76,6 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be annPatches, err = jsonpatch.MergePatch(annPatches, annPatch) if err != nil { glog.Error(err) - fmt.Println("Mergining docs") - fmt.Println(err) } } } @@ -96,7 +92,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be if len(annPatches) > 0 { patches, err = jsonpatch.MergePatch(patches, annPatches) if err != nil { - fmt.Println(err) + glog.Error(err) } } patchType := v1beta1.PatchTypeJSONPatch diff --git a/pkg/webhooks/report.go b/pkg/webhooks/report.go index 4850cd4a77..ef57cb7a78 100644 --- a/pkg/webhooks/report.go +++ b/pkg/webhooks/report.go @@ -1,6 +1,7 @@ package webhooks import ( + "fmt" "strings" "github.com/nirmata/kyverno/pkg/annotations" @@ -63,8 +64,12 @@ func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool } func addAnnotationsToResource(rawResource []byte, pi *info.PolicyInfo, ruleType info.RuleType) []byte { + if len(pi.Rules) == 0 { + return nil + } // get annotations ann := annotations.ParseAnnotationsFromObject(rawResource) + fmt.Println(ann) ann, patch, err := annotations.AddPolicyJSONPatch(ann, pi, ruleType) if err != nil { glog.Error(err) diff --git a/pkg/webhooks/validation.go b/pkg/webhooks/validation.go index 7ee7f20363..733a10eccb 100644 --- a/pkg/webhooks/validation.go +++ b/pkg/webhooks/validation.go @@ -6,7 +6,6 @@ import ( engine "github.com/nirmata/kyverno/pkg/engine" "github.com/nirmata/kyverno/pkg/event" "github.com/nirmata/kyverno/pkg/info" - "github.com/nirmata/kyverno/pkg/violation" v1beta1 "k8s.io/api/admission/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -16,7 +15,7 @@ import ( // If there are no errors in validating rule we apply generation rules func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { policyInfos := []*info.PolicyInfo{} - var violations []*violation.Info + // var violations []*violation.Info var eventsInfo []*event.Info policies, err := ws.policyLister.List(labels.NewSelector()) if err != nil { @@ -27,6 +26,7 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 Allowed: true, } } + rname := engine.ParseNameFromObject(request.Object.Raw) rns := engine.ParseNamespaceFromObject(request.Object.Raw) rkind := engine.ParseKindFromObject(request.Object.Raw) @@ -64,18 +64,18 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 glog.Warning(r.Msgs) } } else { - // CleanUp Violations if exists - err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Validation) - if err != nil { - glog.Info(err) - } + // // CleanUp Violations if exists + // err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Validation) + // if err != nil { + // glog.Info(err) + // } if len(ruleInfos) > 0 { glog.Infof("Validation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) } } policyInfos = append(policyInfos, policyInfo) - annPatch := addAnnotationsToResource(request.Object.Raw, policyInfo, info.Mutation) + annPatch := addAnnotationsToResource(request.Object.Raw, policyInfo, info.Validation) if annPatch != nil { if annPatches == nil { annPatches = annPatch @@ -89,13 +89,13 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 } if len(policyInfos) > 0 && len(policyInfos[0].Rules) != 0 { - eventsInfo, violations = newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update), true) + eventsInfo, _ = newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update), true) // If the validationFailureAction flag is set "report", // then we dont block the request and report the violations - ws.violationBuilder.Add(violations...) + // ws.violationBuilder.Add(violations...) ws.eventController.Add(eventsInfo...) } - // add annotations + // add annotations if annPatches != nil { ws.annotationsController.Add(rkind, rns, rname, annPatches) } From f9b5ac9a27341315fcfaac91e7f8d6468819e527 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Thu, 18 Jul 2019 10:22:20 -0700 Subject: [PATCH 13/24] flag, violations --- definitions/install.yaml | 14 +- pkg/annotations/controller.go | 2 +- pkg/apis/policy/v1alpha1/utils.go | 20 + pkg/controller/controller.go | 12 +- pkg/engine/engine.go | 4 + pkg/info/info.go | 3 + pkg/violation/builder.go | 584 ++++++++++++++++++++---------- pkg/webhooks/deleteresource.go | 8 +- pkg/webhooks/mutation.go | 4 +- pkg/webhooks/report.go | 4 +- pkg/webhooks/utils.go | 2 + pkg/webhooks/validation.go | 17 +- 12 files changed, 452 insertions(+), 222 deletions(-) diff --git a/definitions/install.yaml b/definitions/install.yaml index 34f3ba78c5..22c7806ba4 100644 --- a/definitions/install.yaml +++ b/definitions/install.yaml @@ -21,16 +21,13 @@ spec: spec: required: - rules - # set as required, as we cannot set default yet. check below for more details - - mode properties: - mode: + # default values to be handled by user + validationFailureAction: type: string - # default can only be set if CustomResourceDefaulting feature gate is enabled - # default: blockChanges enum: - - blockChanges - - reportViolation + - block + - report rules: type: array items: @@ -186,7 +183,8 @@ spec: serviceAccountName: kyverno-service-account containers: - name: kyverno - image: nirmata/kyverno:dev-testing + image: nirmata/kyverno:latest +# image: registry-v2.nirmata.io/nirmata/kyverno:testImage args: ["--filterKind","Nodes,Events,APIService,SubjectAccessReview"] ports: - containerPort: 443 diff --git a/pkg/annotations/controller.go b/pkg/annotations/controller.go index cfdb5ec12a..222d91c688 100644 --- a/pkg/annotations/controller.go +++ b/pkg/annotations/controller.go @@ -88,7 +88,7 @@ func (pc *controller) handleErr(err error, key interface{}) { } pc.queue.Forget(key) glog.Error(err) - glog.Warningf("Dropping the key %q out of the queue: %v", key, err) + glog.Warningf("Dropping the key out of the queue: %v", err) } func (c *controller) syncHandler(obj interface{}) error { diff --git a/pkg/apis/policy/v1alpha1/utils.go b/pkg/apis/policy/v1alpha1/utils.go index 17c067c5fb..0c2c8fa00e 100644 --- a/pkg/apis/policy/v1alpha1/utils.go +++ b/pkg/apis/policy/v1alpha1/utils.go @@ -105,6 +105,26 @@ func (in *Generation) DeepCopyInto(out *Generation) { } } +// return true -> if there were any removals +// return false -> if it looks the same +func (v *Violation) RemoveRulesOfType(ruleType string) bool { + removed := false + updatedRules := []FailedRule{} + for _, r := range v.Rules { + if r.Type == ruleType { + removed = true + continue + } + updatedRules = append(updatedRules, r) + } + + if removed { + v.Rules = updatedRules + return true + } + return false +} + //IsEqual Check if violatiosn are equal func (v *Violation) IsEqual(nv Violation) bool { // We do not need to compare resource info as it will be same diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index bc14db812c..dc7a1ee8df 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -162,7 +162,7 @@ func (pc *PolicyController) handleErr(err error, key interface{}) { } pc.queue.Forget(key) glog.Error(err) - glog.Warningf("Dropping the key %q out of the queue: %v", key, err) + glog.Warningf("Dropping the key out of the queue: %v", err) } func (pc *PolicyController) syncHandler(obj interface{}) error { @@ -192,12 +192,12 @@ func (pc *PolicyController) syncHandler(obj interface{}) error { //TODO: processPolicy glog.Infof("process policy %s on existing resources", policy.GetName()) policyInfos := engine.ProcessExisting(pc.client, policy) - events, _ := pc.createEventsAndViolations(policyInfos) + events, violations := pc.createEventsAndViolations(policyInfos) pc.eventController.Add(events...) - // err = pc.violationBuilder.Add(violations...) - // if err != nil { - // glog.Error(err) - // } + err = pc.violationBuilder.Add(violations...) + if err != nil { + glog.Error(err) + } // add annotations to resources pc.createAnnotations(policyInfos) diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 79142d9269..ed938048bc 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -101,6 +101,10 @@ func applyPolicy(client *client.Client, policy *types.Policy, res *resourceInfo) func mutation(p *types.Policy, rawResource []byte, gvk *metav1.GroupVersionKind) ([]*info.RuleInfo, error) { patches, ruleInfos := Mutate(*p, rawResource, *gvk) + if len(ruleInfos) == 0 { + // no rules were processed + return nil, nil + } // option 2: (original Resource + patch) compare with (original resource) mergePatches := JoinPatches(patches) // merge the patches diff --git a/pkg/info/info.go b/pkg/info/info.go index 9f0dc004c0..260dbb35e0 100644 --- a/pkg/info/info.go +++ b/pkg/info/info.go @@ -159,6 +159,9 @@ func RulesSuccesfuly(rules []*RuleInfo) bool { //AddRuleInfos sets the rule information func (pi *PolicyInfo) AddRuleInfos(rules []*RuleInfo) { + if rules == nil { + return + } if !RulesSuccesfuly(rules) { pi.success = false } diff --git a/pkg/violation/builder.go b/pkg/violation/builder.go index e15bf88e05..75514e35ec 100644 --- a/pkg/violation/builder.go +++ b/pkg/violation/builder.go @@ -12,6 +12,7 @@ import ( event "github.com/nirmata/kyverno/pkg/event" "github.com/nirmata/kyverno/pkg/info" "github.com/nirmata/kyverno/pkg/sharedinformer" + "k8s.io/apimachinery/pkg/runtime" ) //Generator to generate policy violation @@ -48,6 +49,9 @@ func NewPolicyViolationBuilder(client *client.Client, } func (b *builder) Add(infos ...*Info) error { + if infos == nil { + return nil + } for _, info := range infos { return b.processViolation(info) } @@ -55,97 +59,297 @@ func (b *builder) Add(infos ...*Info) error { } func (b *builder) processViolation(info *Info) error { - currVs := map[string]interface{}{} statusMap := map[string]interface{}{} - var ok bool - //TODO: hack get from client - p1, err := b.client.GetResource("Policy", "", info.Policy, "status") + violationsMap := map[string]interface{}{} + violationMap := map[string]interface{}{} + var violations interface{} + var violation interface{} + // Get Policy + obj, err := b.client.GetResource("Policy", "", info.Policy, "status") if err != nil { return err } - unstr := p1.UnstructuredContent() - // check if "status" field exists + unstr := obj.UnstructuredContent() + // get "status" subresource status, ok := unstr["status"] if ok { + // status exists // status is already present then we append violations if statusMap, ok = status.(map[string]interface{}); !ok { return errors.New("Unable to parse status subresource") } - violations, ok := statusMap["violations"] + // get policy violations + violations, ok = statusMap["violations"] if !ok { - glog.Info("violation not present") - } - // Violations map[string][]Violations - glog.Info(reflect.TypeOf(violations)) - if currVs, ok = violations.(map[string]interface{}); !ok { - return errors.New("Unable to parse violations") - } - } - // Info: - // Key - Kind, Namespace, Name - // policy - Name - // violation, ok := currVs[info.getKey()] - // Key -> resource - // 1> Check if there were any previous violations for the given key - // 2> If No, create a new one - if !ok { - currVs[info.getKey()] = info.Violation - } else { - currV := currVs[info.getKey()] - glog.Info(reflect.TypeOf(currV)) - v, ok := currV.(map[string]interface{}) - if !ok { - glog.Info("type not matching") - } - // get rules - rules, ok := v["rules"] - if !ok { - glog.Info("rules not found") - } - glog.Info(reflect.TypeOf(rules)) - rs, ok := rules.([]interface{}) - if !ok { - glog.Info("type not matching") - } - // check if rules are samre - if isRuleNamesEqual(rs, info.Violation.Rules) { return nil } - // else update the errors - currVs[info.getKey()] = info.Violation + violationsMap, ok = violations.(map[string]interface{}) + if !ok { + return errors.New("Unable to get status.violations subresource") + } + // check if the resource has a violation + violation, ok = violationsMap[info.getKey()] + if !ok { + // add resource violation + violationsMap[info.getKey()] = info.Violation + statusMap["violations"] = violationsMap + unstr["status"] = statusMap + } else { + violationMap, ok = violation.(map[string]interface{}) + if !ok { + return errors.New("Unable to get status.violations.violation subresource") + } + // we check if the new violation updates are different from stored violation info + v := v1alpha1.Violation{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(violationMap, &v) + if err != nil { + return err + } + // compare v & info.Violation + if v.IsEqual(info.Violation) { + // no updates to violation + // do nothing + return nil + } + // update the violation + violationsMap[info.getKey()] = info.Violation + statusMap["violations"] = violationsMap + unstr["status"] = statusMap + } + } else { + violationsMap[info.getKey()] = info.Violation + statusMap["violations"] = violationsMap + unstr["status"] = statusMap } - // newViolation := info.Violation - // for _, violation := range currentViolations { - // glog.Info(reflect.TypeOf(violation)) - // if v, ok := violation.(map[string]interface{}); ok { - // if name, ok := v["name"].(string); ok { - // if namespace, ok := v["namespace"].(string); ok { - // ok, err := b.isActive(info.Kind, name, namespace) - // if err != nil { - // glog.Error(err) - // continue - // } - // if !ok { - // //TODO remove the violation as it corresponds to resource that does not exist - // glog.Info("removed violation") - // } - // } - // } - // } - // } - // currentViolations = append(currentViolations, newViolation) - // // update violations - // set the updated status - statusMap["violations"] = currVs - unstr["status"] = statusMap - p1.SetUnstructuredContent(unstr) - _, err = b.client.UpdateStatusResource("Policy", "", p1, false) + + obj.SetUnstructuredContent(unstr) + // update the status sub-resource for policy + _, err = b.client.UpdateStatusResource("Policy", "", obj, false) if err != nil { return err } return nil } +func (b *builder) RemoveInactiveViolation(policy, rKind, rNs, rName string, ruleType info.RuleType) error { + statusMap := map[string]interface{}{} + violationsMap := map[string]interface{}{} + violationMap := map[string]interface{}{} + var violations interface{} + var violation interface{} + // Get Policy + obj, err := b.client.GetResource("Policy", "", policy, "status") + if err != nil { + return err + } + unstr := obj.UnstructuredContent() + // get "status" subresource + status, ok := unstr["status"] + if !ok { + return nil + } + // status exists + // status is already present then we append violations + if statusMap, ok = status.(map[string]interface{}); !ok { + return errors.New("Unable to parse status subresource") + } + // get policy violations + violations, ok = statusMap["violations"] + if !ok { + return nil + } + violationsMap, ok = violations.(map[string]interface{}) + if !ok { + return errors.New("Unable to get status.violations subresource") + } + // check if the resource has a violation + violation, ok = violationsMap[BuildKey(rKind, rNs, rName)] + if !ok { + // no violation for this resource + return nil + } + violationMap, ok = violation.(map[string]interface{}) + if !ok { + return errors.New("Unable to get status.violations.violation subresource") + } + // check remove the rules of the given type + // this is called when the policy is applied succesfully, so we can remove the previous failed rules + // if all rules are to be removed, the deleted the violation + v := v1alpha1.Violation{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(violationMap, &v) + if err != nil { + return err + } + if !v.RemoveRulesOfType(ruleType.String()) { + // no rule of given type found, + // no need to remove rule + return nil + } + // if there are no faile rules remove the violation + if len(v.Rules) == 0 { + delete(violationsMap, BuildKey(rKind, rNs, rName)) + } else { + // update the rules + violationsMap[BuildKey(rKind, rNs, rName)] = v + } + statusMap["violations"] = violationsMap + unstr["status"] = statusMap + + obj.SetUnstructuredContent(unstr) + // update the status sub-resource for policy + _, err = b.client.UpdateStatusResource("Policy", "", obj, false) + if err != nil { + return err + } + return nil +} + +func (b *builder) ResourceRemoval(policy, rKind, rNs, rName string) error { + statusMap := map[string]interface{}{} + violationsMap := map[string]interface{}{} + var violations interface{} + // Get Policy + obj, err := b.client.GetResource("Policy", "", policy, "status") + if err != nil { + return err + } + unstr := obj.UnstructuredContent() + // get "status" subresource + status, ok := unstr["status"] + if !ok { + return nil + } + // status exists + // status is already present then we append violations + if statusMap, ok = status.(map[string]interface{}); !ok { + return errors.New("Unable to parse status subresource") + } + // get policy violations + violations, ok = statusMap["violations"] + if !ok { + return nil + } + violationsMap, ok = violations.(map[string]interface{}) + if !ok { + return errors.New("Unable to get status.violations subresource") + } + + // check if the resource has a violation + _, ok = violationsMap[BuildKey(rKind, rNs, rName)] + if !ok { + // no violation for this resource + return nil + } + // remove the pair from the map + delete(violationsMap, BuildKey(rKind, rNs, rName)) + if len(violationsMap) == 0 { + delete(statusMap, "violations") + } else { + statusMap["violations"] = violationsMap + } + unstr["status"] = statusMap + + obj.SetUnstructuredContent(unstr) + // update the status sub-resource for policy + _, err = b.client.UpdateStatusResource("Policy", "", obj, false) + if err != nil { + return err + } + return nil +} + +// func (b *builder) processViolation(info *Info) error { +// currVs := map[string]interface{}{} +// statusMap := map[string]interface{}{} +// var ok bool +// //TODO: hack get from client +// p1, err := b.client.GetResource("Policy", "", info.Policy, "status") +// if err != nil { +// return err +// } +// unstr := p1.UnstructuredContent() +// // check if "status" field exists +// status, ok := unstr["status"] +// if ok { +// // status is already present then we append violations +// if statusMap, ok = status.(map[string]interface{}); !ok { +// return errors.New("Unable to parse status subresource") +// } +// violations, ok := statusMap["violations"] +// if !ok { +// glog.Info("violation not present") +// } +// // Violations map[string][]Violations +// glog.Info(reflect.TypeOf(violations)) +// if currVs, ok = violations.(map[string]interface{}); !ok { +// return errors.New("Unable to parse violations") +// } +// } +// // Info: +// // Key - Kind, Namespace, Name +// // policy - Name +// // violation, ok := currVs[info.getKey()] +// // Key -> resource +// // 1> Check if there were any previous violations for the given key +// // 2> If No, create a new one +// if !ok { +// currVs[info.getKey()] = info.Violation +// } else { +// currV := currVs[info.getKey()] +// glog.Info(reflect.TypeOf(currV)) +// v, ok := currV.(map[string]interface{}) +// if !ok { +// glog.Info("type not matching") +// } +// // get rules +// rules, ok := v["rules"] +// if !ok { +// glog.Info("rules not found") +// } +// glog.Info(reflect.TypeOf(rules)) +// rs, ok := rules.([]interface{}) +// if !ok { +// glog.Info("type not matching") +// } +// // check if rules are samre +// if isRuleNamesEqual(rs, info.Violation.Rules) { +// return nil +// } +// // else update the errors +// currVs[info.getKey()] = info.Violation +// } +// // newViolation := info.Violation +// // for _, violation := range currentViolations { +// // glog.Info(reflect.TypeOf(violation)) +// // if v, ok := violation.(map[string]interface{}); ok { +// // if name, ok := v["name"].(string); ok { +// // if namespace, ok := v["namespace"].(string); ok { +// // ok, err := b.isActive(info.Kind, name, namespace) +// // if err != nil { +// // glog.Error(err) +// // continue +// // } +// // if !ok { +// // //TODO remove the violation as it corresponds to resource that does not exist +// // glog.Info("removed violation") +// // } +// // } +// // } +// // } +// // } +// // currentViolations = append(currentViolations, newViolation) +// // // update violations +// // set the updated status +// statusMap["violations"] = currVs +// unstr["status"] = statusMap +// p1.SetUnstructuredContent(unstr) +// _, err = b.client.UpdateStatusResource("Policy", "", p1, false) +// if err != nil { +// return err +// } +// return nil +// } + func (b *builder) isActive(kind, rname, rnamespace string) (bool, error) { // Generate Merge Patch _, err := b.client.GetResource(b.client.DiscoveryClient.GetGVRFromKind(kind).Resource, rnamespace, rname) @@ -254,124 +458,124 @@ func isRuleNamesEqual(currRules []interface{}, newRules []v1alpha1.FailedRule) b return true } -//RemoveViolation will remove the violation for the resource if there was one -func (b *builder) RemoveInactiveViolation(policy, rKind, rNs, rName string, ruleType info.RuleType) error { - // Remove the pair from map - statusMap := map[string]interface{}{} - currVs := map[string]interface{}{} - // Get the policy - p, err := b.client.GetResource("Policy", "", policy, "status") - if err != nil { - glog.Infof("policy %s not found", policy) - return err - } - unstr := p.UnstructuredContent() +// //RemoveViolation will remove the violation for the resource if there was one +// func (b *builder) RemoveInactiveViolation(policy, rKind, rNs, rName string, ruleType info.RuleType) error { +// // Remove the pair from map +// statusMap := map[string]interface{}{} +// currVs := map[string]interface{}{} +// // Get the policy +// p, err := b.client.GetResource("Policy", "", policy, "status") +// if err != nil { +// glog.Infof("policy %s not found", policy) +// return err +// } +// unstr := p.UnstructuredContent() - // check if "status" field exists - status, ok := unstr["status"] - if ok { - // status is already present then we append violations - if statusMap, ok = status.(map[string]interface{}); !ok { - return errors.New("Unable to parse status subresource") - } - violations, ok := statusMap["violations"] - if !ok { - glog.Info("violation not present") - } - glog.Info(reflect.TypeOf(violations)) - if currVs, ok = violations.(map[string]interface{}); !ok { - return errors.New("Unable to parse violations") - } - currV, ok := currVs[BuildKey(rKind, rNs, rName)] - if !ok { - // No Violation present - return nil - } - glog.Info(reflect.TypeOf(currV)) - v, ok := currV.(map[string]interface{}) - if !ok { - glog.Info("type not matching") - } - // get rules - rules, ok := v["rules"] - if !ok { - glog.Info("rules not found") - } - glog.Info(reflect.TypeOf(rules)) - rs, ok := rules.([]interface{}) - if !ok { - glog.Info("type not matching") - } - // Remove rules of defined type - newrs, err := removeRuleTypes(rs, ruleType) - if err != nil { - glog.Info(err) - } - if newrs == nil { - // all records are removed and is empty - glog.Info("can remove the record") - delete(currVs, BuildKey(rKind, rNs, rName)) - } else { - v["rules"] = newrs - // update the violation with new rule - currVs[BuildKey(rKind, rNs, rName)] = v - } - // update violations - statusMap["violations"] = currVs - // update status - unstr["status"] = statusMap - p.SetUnstructuredContent(unstr) - _, err = b.client.UpdateStatusResource("Policy", "", p, false) - if err != nil { - return err - } - } - return nil -} +// // check if "status" field exists +// status, ok := unstr["status"] +// if ok { +// // status is already present then we append violations +// if statusMap, ok = status.(map[string]interface{}); !ok { +// return errors.New("Unable to parse status subresource") +// } +// violations, ok := statusMap["violations"] +// if !ok { +// glog.Info("violation not present") +// } +// glog.Info(reflect.TypeOf(violations)) +// if currVs, ok = violations.(map[string]interface{}); !ok { +// return errors.New("Unable to parse violations") +// } +// currV, ok := currVs[BuildKey(rKind, rNs, rName)] +// if !ok { +// // No Violation present +// return nil +// } +// glog.Info(reflect.TypeOf(currV)) +// v, ok := currV.(map[string]interface{}) +// if !ok { +// glog.Info("type not matching") +// } +// // get rules +// rules, ok := v["rules"] +// if !ok { +// glog.Info("rules not found") +// } +// glog.Info(reflect.TypeOf(rules)) +// rs, ok := rules.([]interface{}) +// if !ok { +// glog.Info("type not matching") +// } +// // Remove rules of defined type +// newrs, err := removeRuleTypes(rs, ruleType) +// if err != nil { +// glog.Info(err) +// } +// if newrs == nil { +// // all records are removed and is empty +// glog.Info("can remove the record") +// delete(currVs, BuildKey(rKind, rNs, rName)) +// } else { +// v["rules"] = newrs +// // update the violation with new rule +// currVs[BuildKey(rKind, rNs, rName)] = v +// } +// // update violations +// statusMap["violations"] = currVs +// // update status +// unstr["status"] = statusMap +// p.SetUnstructuredContent(unstr) +// _, err = b.client.UpdateStatusResource("Policy", "", p, false) +// if err != nil { +// return err +// } +// } +// return nil +// } -func (b *builder) ResourceRemoval(policy, rKind, rNs, rName string) error { - // Remove the pair from map - statusMap := map[string]interface{}{} - currVs := map[string]interface{}{} - // Get the policy - p, err := b.client.GetResource("Policy", "", policy, "status") - if err != nil { - glog.Infof("policy %s not found", policy) - return err - } - unstr := p.UnstructuredContent() - // check if "status" field exists - status, ok := unstr["status"] - if ok { - // status is already present then we append violations - if statusMap, ok = status.(map[string]interface{}); !ok { - return errors.New("Unable to parse status subresource") - } - violations, ok := statusMap["violations"] - if !ok { - glog.Info("violation not present") - } - glog.Info(reflect.TypeOf(violations)) - if currVs, ok = violations.(map[string]interface{}); !ok { - return errors.New("Unable to parse violations") - } - _, ok = currVs[BuildKey(rKind, rNs, rName)] - if !ok { - // No Violation for this resource - return nil - } - // remove the pair from the map - delete(currVs, BuildKey(rKind, rNs, rName)) - glog.Info("Removed Violation") - // update violations - statusMap["violations"] = currVs - // update status - unstr["status"] = statusMap - p.SetUnstructuredContent(unstr) - _, err = b.client.UpdateStatusResource("Policy", "", p, false) - if err != nil { - return err - } - } - return nil -} +// func (b *builder) ResourceRemoval(policy, rKind, rNs, rName string) error { +// // Remove the pair from map +// statusMap := map[string]interface{}{} +// currVs := map[string]interface{}{} +// // Get the policy +// p, err := b.client.GetResource("Policy", "", policy, "status") +// if err != nil { +// glog.Infof("policy %s not found", policy) +// return err +// } +// unstr := p.UnstructuredContent() +// // check if "status" field exists +// status, ok := unstr["status"] +// if ok { +// // status is already present then we append violations +// if statusMap, ok = status.(map[string]interface{}); !ok { +// return errors.New("Unable to parse status subresource") +// } +// violations, ok := statusMap["violations"] +// if !ok { +// glog.Info("violation not present") +// } +// glog.Info(reflect.TypeOf(violations)) +// if currVs, ok = violations.(map[string]interface{}); !ok { +// return errors.New("Unable to parse violations") +// } +// _, ok = currVs[BuildKey(rKind, rNs, rName)] +// if !ok { +// // No Violation for this resource +// return nil +// } +// // remove the pair from the map +// delete(currVs, BuildKey(rKind, rNs, rName)) +// glog.Info("Removed Violation") +// // update violations +// statusMap["violations"] = currVs +// // update status +// unstr["status"] = statusMap +// p.SetUnstructuredContent(unstr) +// _, err = b.client.UpdateStatusResource("Policy", "", p, false) +// if err != nil { +// return err +// } +// } +// return nil +// } diff --git a/pkg/webhooks/deleteresource.go b/pkg/webhooks/deleteresource.go index a1449791db..eb39e6df36 100644 --- a/pkg/webhooks/deleteresource.go +++ b/pkg/webhooks/deleteresource.go @@ -9,7 +9,6 @@ import ( ) func (ws *WebhookServer) removePolicyViolation(request *v1beta1.AdmissionRequest) error { - return nil // Get the list of policies that apply on the resource policies, err := ws.policyLister.List(labels.NewSelector()) if err != nil { @@ -21,9 +20,10 @@ func (ws *WebhookServer) removePolicyViolation(request *v1beta1.AdmissionRequest if !StringInSlice(request.Kind.Kind, getApplicableKindsForPolicy(policy)) { continue } - rname := engine.ParseNameFromObject(request.Object.Raw) - rns := engine.ParseNamespaceFromObject(request.Object.Raw) - rkind := engine.ParseKindFromObject(request.Object.Raw) + // get the details from the request + rname := request.Name + rns := request.Namespace + rkind := request.Kind.Kind // check if the resource meets the policy Resource description for _, rule := range policy.Spec.Rules { ok := engine.ResourceMeetsDescription(request.Object.Raw, rule.ResourceDescription, request.Kind) diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go index 957b5abf79..3028d9fbed 100644 --- a/pkg/webhooks/mutation.go +++ b/pkg/webhooks/mutation.go @@ -56,7 +56,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be glog.Warning(r.Msgs) } } else { - // CleanUp Violations if exists + // // CleanUp Violations if exists // err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Mutation) // if err != nil { // glog.Info(err) @@ -82,7 +82,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be } if len(allPatches) > 0 { - eventsInfo, _ := newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update), false) + eventsInfo, _ := newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update), info.Mutation) ws.eventController.Add(eventsInfo...) } diff --git a/pkg/webhooks/report.go b/pkg/webhooks/report.go index ef57cb7a78..a80667db41 100644 --- a/pkg/webhooks/report.go +++ b/pkg/webhooks/report.go @@ -13,7 +13,7 @@ import ( ) //TODO: change validation from bool -> enum(validation, mutation) -func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool, validation bool) ([]*event.Info, []*violation.Info) { +func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool, ruleType info.RuleType) ([]*event.Info, []*violation.Info) { var eventsInfo []*event.Info var violations []*violation.Info ok, msg := isAdmSuccesful(policyInfoList) @@ -40,7 +40,7 @@ func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool glog.V(3).Infof("Request blocked events info has prepared for %s/%s and %s/%s\n", policyKind, pi.Name, pi.RKind, pi.RName) } // if report flag is set - if pi.ValidationFailureAction == ReportViolation && validation { + if pi.ValidationFailureAction == ReportViolation && ruleType == info.Validation { // Create Violations v := violation.BuldNewViolation(pi.Name, pi.RKind, pi.RNamespace, pi.RName, event.PolicyViolation.String(), pi.GetFailedRules()) violations = append(violations, v) diff --git a/pkg/webhooks/utils.go b/pkg/webhooks/utils.go index e70fc448c3..07e5b86fc6 100644 --- a/pkg/webhooks/utils.go +++ b/pkg/webhooks/utils.go @@ -89,6 +89,8 @@ const ( ReportViolation = "report" ) +// returns true -> if there is even one policy that blocks resource requst +// returns false -> if all the policies are meant to report only, we dont block resource request func toBlock(pis []*info.PolicyInfo) bool { for _, pi := range pis { if pi.ValidationFailureAction != ReportViolation { diff --git a/pkg/webhooks/validation.go b/pkg/webhooks/validation.go index 733a10eccb..8e3ecfa5c6 100644 --- a/pkg/webhooks/validation.go +++ b/pkg/webhooks/validation.go @@ -4,7 +4,6 @@ import ( jsonpatch "github.com/evanphx/json-patch" "github.com/golang/glog" engine "github.com/nirmata/kyverno/pkg/engine" - "github.com/nirmata/kyverno/pkg/event" "github.com/nirmata/kyverno/pkg/info" v1beta1 "k8s.io/api/admission/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -16,7 +15,7 @@ import ( func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { policyInfos := []*info.PolicyInfo{} // var violations []*violation.Info - var eventsInfo []*event.Info + // var eventsInfo []*event.Info policies, err := ws.policyLister.List(labels.NewSelector()) if err != nil { // Unable to connect to policy Lister to access policies @@ -64,11 +63,11 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 glog.Warning(r.Msgs) } } else { - // // CleanUp Violations if exists - // err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Validation) - // if err != nil { - // glog.Info(err) - // } + // CleanUp Violations if exists + err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Validation) + if err != nil { + glog.Info(err) + } if len(ruleInfos) > 0 { glog.Infof("Validation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) @@ -89,10 +88,10 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 } if len(policyInfos) > 0 && len(policyInfos[0].Rules) != 0 { - eventsInfo, _ = newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update), true) + eventsInfo, violations := newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update), info.Validation) // If the validationFailureAction flag is set "report", // then we dont block the request and report the violations - // ws.violationBuilder.Add(violations...) + ws.violationBuilder.Add(violations...) ws.eventController.Add(eventsInfo...) } // add annotations From 1d23dbbbb894b1df464c4c0ebceabdea925d2f7d Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Thu, 18 Jul 2019 15:49:29 -0700 Subject: [PATCH 14/24] update annotation key --- pkg/annotations/annotations.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 360debe536..65b155fe9e 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -32,6 +32,10 @@ func getStatus(status bool) string { return "Failure" } +func buildKey(policyName string) string { + return "policies.kyverno.io." + policyName +} + func getRules(rules []*pinfo.RuleInfo, ruleType pinfo.RuleType) map[string]Rule { if len(rules) == 0 { return nil @@ -146,7 +150,7 @@ func AddPolicy(obj *unstructured.Unstructured, pi *pinfo.PolicyInfo, ruleType pi // get annotation ann := obj.GetAnnotations() // check if policy already has annotation - cPolicy, ok := ann[pi.Name] + cPolicy, ok := ann[buildKey(pi.Name)] if !ok { PolicyByte, err := json.Marshal(PolicyObj) if err != nil { @@ -154,7 +158,7 @@ func AddPolicy(obj *unstructured.Unstructured, pi *pinfo.PolicyInfo, ruleType pi return false } // insert policy information - ann[pi.Name] = string(PolicyByte) + ann[buildKey(pi.Name)] = string(PolicyByte) // set annotation back to unstr obj.SetAnnotations(ann) return true @@ -173,7 +177,7 @@ func AddPolicy(obj *unstructured.Unstructured, pi *pinfo.PolicyInfo, ruleType pi return false } // update policy information - ann[pi.Name] = string(cPolicyByte) + ann[buildKey(pi.Name)] = string(cPolicyByte) // set annotation back to unstr obj.SetAnnotations(ann) return true @@ -190,10 +194,10 @@ func RemovePolicy(obj *unstructured.Unstructured, policy string) bool { if ann == nil { return false } - if _, ok := ann[policy]; !ok { + if _, ok := ann[buildKey(policy)]; !ok { return false } - delete(ann, policy) + delete(ann, buildKey(policy)) // set annotation back to unstr obj.SetAnnotations(ann) return true @@ -221,14 +225,14 @@ func AddPolicyJSONPatch(ann map[string]string, pi *pinfo.PolicyInfo, ruleType pi ann = make(map[string]string, 0) } PolicyObj := newAnnotationForPolicy(pi) - cPolicy, ok := ann[pi.Name] + cPolicy, ok := ann[buildKey(pi.Name)] if !ok { PolicyByte, err := json.Marshal(PolicyObj) if err != nil { return nil, nil, err } // insert policy information - ann[pi.Name] = string(PolicyByte) + ann[buildKey(pi.Name)] = string(PolicyByte) // create add JSON patch jsonPatch, err := createAddJSONPatch(ann) // var jsonPatch []byte @@ -253,7 +257,7 @@ func AddPolicyJSONPatch(ann map[string]string, pi *pinfo.PolicyInfo, ruleType pi return nil, nil, err } // update policy information - ann[pi.Name] = string(cPolicyByte) + ann[buildKey(pi.Name)] = string(cPolicyByte) // create update JSON patch jsonPatch, err := createReplaceJSONPatch(ann) return ann, jsonPatch, err From 9fcb4b7b10f47b670516c0cab01618fd4ed44d24 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Fri, 19 Jul 2019 12:47:20 -0700 Subject: [PATCH 15/24] bypass annotation additions --- pkg/webhooks/mutation.go | 6 ++++++ pkg/webhooks/utils.go | 31 +++++++++++++++++++++++++++++++ pkg/webhooks/validation.go | 7 +++++++ 3 files changed, 44 insertions(+) diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go index 3028d9fbed..9dd39b6bf0 100644 --- a/pkg/webhooks/mutation.go +++ b/pkg/webhooks/mutation.go @@ -32,6 +32,12 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be if !StringInSlice(request.Kind.Kind, getApplicableKindsForPolicy(policy)) { continue } + //TODO: HACK Check if an update of annotations + if checkIfOnlyAnnotationsUpdate(request) { + return &v1beta1.AdmissionResponse{ + Allowed: true, + } + } rname := engine.ParseNameFromObject(request.Object.Raw) rns := engine.ParseNamespaceFromObject(request.Object.Raw) rkind := engine.ParseKindFromObject(request.Object.Raw) diff --git a/pkg/webhooks/utils.go b/pkg/webhooks/utils.go index 07e5b86fc6..f2ac23a441 100644 --- a/pkg/webhooks/utils.go +++ b/pkg/webhooks/utils.go @@ -2,10 +2,14 @@ package webhooks import ( "fmt" + "reflect" "strings" + "github.com/golang/glog" "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" "github.com/nirmata/kyverno/pkg/info" + v1beta1 "k8s.io/api/admission/v1beta1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) const policyKind = "Policy" @@ -99,3 +103,30 @@ func toBlock(pis []*info.PolicyInfo) bool { } return false } + +func checkIfOnlyAnnotationsUpdate(request *v1beta1.AdmissionRequest) bool { + // updated resoruce + obj := request.Object + objUnstr := unstructured.Unstructured{} + err := objUnstr.UnmarshalJSON(obj.Raw) + if err != nil { + glog.Error(err) + return false + } + objUnstr.SetAnnotations(nil) + objUnstr.SetGeneration(0) + oldobj := request.OldObject + oldobjUnstr := unstructured.Unstructured{} + err = oldobjUnstr.UnmarshalJSON(oldobj.Raw) + if err != nil { + glog.Error(err) + return false + } + oldobjUnstr.SetAnnotations(nil) + oldobjUnstr.SetGeneration(0) + if reflect.DeepEqual(objUnstr, oldobjUnstr) { + return true + } + + return false +} diff --git a/pkg/webhooks/validation.go b/pkg/webhooks/validation.go index 8e3ecfa5c6..b8d5eaa91d 100644 --- a/pkg/webhooks/validation.go +++ b/pkg/webhooks/validation.go @@ -36,6 +36,13 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 if !StringInSlice(request.Kind.Kind, getApplicableKindsForPolicy(policy)) { continue } + //TODO: HACK Check if an update of annotations + if checkIfOnlyAnnotationsUpdate(request) { + // allow the update of resource to add annotations + return &v1beta1.AdmissionResponse{ + Allowed: true, + } + } policyInfo := info.NewPolicyInfo(policy.Name, rkind, From d68c4ea033d3cfab9c8956e8c12b4353981b212b Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Fri, 19 Jul 2019 13:53:36 -0700 Subject: [PATCH 16/24] check for annotattions for resource updates only --- pkg/webhooks/utils.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/webhooks/utils.go b/pkg/webhooks/utils.go index f2ac23a441..b29cd16449 100644 --- a/pkg/webhooks/utils.go +++ b/pkg/webhooks/utils.go @@ -105,6 +105,10 @@ func toBlock(pis []*info.PolicyInfo) bool { } func checkIfOnlyAnnotationsUpdate(request *v1beta1.AdmissionRequest) bool { + // process only if its for existing resources + if request.Operation != v1beta1.Update { + return false + } // updated resoruce obj := request.Object objUnstr := unstructured.Unstructured{} From 9f157544c9c47d3467cc9b2157581ecd04995b5a Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Fri, 19 Jul 2019 15:10:40 -0700 Subject: [PATCH 17/24] cleanUp --- pkg/annotations/annotations.go | 66 +----- pkg/annotations/controller.go | 2 +- pkg/annotations/info.go | 18 ++ pkg/annotations/utils.go | 24 +-- pkg/apis/policy/v1alpha1/types.go | 5 +- pkg/controller/cleanup.go | 3 +- pkg/controller/controller.go | 29 +-- pkg/gencontroller/generation.go | 4 +- pkg/info/info.go | 8 +- pkg/violation/builder.go | 342 ++---------------------------- pkg/violation/util.go | 3 +- pkg/webhooks/validation.go | 3 +- 12 files changed, 78 insertions(+), 429 deletions(-) create mode 100644 pkg/annotations/info.go diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 65b155fe9e..92623805a3 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -23,17 +23,7 @@ type Policy struct { type Rule struct { Status string `json:"status"` Changes string `json:"changes,omitempty"` // TODO for mutation changes -} - -func getStatus(status bool) string { - if status { - return "Success" - } - return "Failure" -} - -func buildKey(policyName string) string { - return "policies.kyverno.io." + policyName + Error string `json:"error,omitempty"` } func getRules(rules []*pinfo.RuleInfo, ruleType pinfo.RuleType) map[string]Rule { @@ -47,9 +37,7 @@ func getRules(rules []*pinfo.RuleInfo, ruleType pinfo.RuleType) map[string]Rule continue } annrules[r.Name] = - Rule{Status: getStatus(r.IsSuccessful())} - // //TODO: add mutation changes in policyInfo and in annotation - // annrules = append(annrules, annrule) + Rule{Status: getStatus(r.IsSuccessful()), Error: r.GetErrorString()} } return annrules } @@ -106,33 +94,6 @@ func (p *Policy) compareGenerationRules(rules map[string]Rule) bool { return false } -// // Update rules of a given type -// func (p *Policy) updatePolicyRules(rules map[string]Rule, ruleType info.RuleType) bool { -// updates := false -// // Check if new rules are present in existing rules -// // for k, v := range rules { -// // _, -// // } - -// var updatedRules []Rule -// //TODO: check the selecting update add advantage -// // filter rules for different type -// for _, r := range rules { -// if r.Type != ruleType.String() { -// updatedRules = append(updatedRules, r) -// } -// } -// // Add rules for current type -// updatedRules = append(updatedRules, rules...) -// // set the rule -// p.Rules = updatedRules -// } - -// func (p *Policy) containsPolicyRules(rules []Rule, ruleType info.RuleType) { -// for _, r := range rules { -// } -// } - func newAnnotationForPolicy(pi *pinfo.PolicyInfo) *Policy { return &Policy{Status: getStatus(pi.IsSuccessful()), MutationRules: getRules(pi.Rules, pinfo.Mutation), @@ -150,7 +111,7 @@ func AddPolicy(obj *unstructured.Unstructured, pi *pinfo.PolicyInfo, ruleType pi // get annotation ann := obj.GetAnnotations() // check if policy already has annotation - cPolicy, ok := ann[buildKey(pi.Name)] + cPolicy, ok := ann[BuildKey(pi.Name)] if !ok { PolicyByte, err := json.Marshal(PolicyObj) if err != nil { @@ -158,7 +119,7 @@ func AddPolicy(obj *unstructured.Unstructured, pi *pinfo.PolicyInfo, ruleType pi return false } // insert policy information - ann[buildKey(pi.Name)] = string(PolicyByte) + ann[BuildKey(pi.Name)] = string(PolicyByte) // set annotation back to unstr obj.SetAnnotations(ann) return true @@ -177,7 +138,7 @@ func AddPolicy(obj *unstructured.Unstructured, pi *pinfo.PolicyInfo, ruleType pi return false } // update policy information - ann[buildKey(pi.Name)] = string(cPolicyByte) + ann[BuildKey(pi.Name)] = string(cPolicyByte) // set annotation back to unstr obj.SetAnnotations(ann) return true @@ -194,10 +155,10 @@ func RemovePolicy(obj *unstructured.Unstructured, policy string) bool { if ann == nil { return false } - if _, ok := ann[buildKey(policy)]; !ok { + if _, ok := ann[BuildKey(policy)]; !ok { return false } - delete(ann, buildKey(policy)) + delete(ann, BuildKey(policy)) // set annotation back to unstr obj.SetAnnotations(ann) return true @@ -225,22 +186,17 @@ func AddPolicyJSONPatch(ann map[string]string, pi *pinfo.PolicyInfo, ruleType pi ann = make(map[string]string, 0) } PolicyObj := newAnnotationForPolicy(pi) - cPolicy, ok := ann[buildKey(pi.Name)] + cPolicy, ok := ann[BuildKey(pi.Name)] if !ok { PolicyByte, err := json.Marshal(PolicyObj) if err != nil { return nil, nil, err } // insert policy information - ann[buildKey(pi.Name)] = string(PolicyByte) + ann[BuildKey(pi.Name)] = string(PolicyByte) // create add JSON patch jsonPatch, err := createAddJSONPatch(ann) - // var jsonPatch []byte - // if len(ann) == 0 { - // jsonPatch, err = createAddJSONPatch(ann) - // } else { - // jsonPatch, err = createReplaceJSONPatch(ann) - // } + return ann, jsonPatch, err } cPolicyObj := Policy{} @@ -257,7 +213,7 @@ func AddPolicyJSONPatch(ann map[string]string, pi *pinfo.PolicyInfo, ruleType pi return nil, nil, err } // update policy information - ann[buildKey(pi.Name)] = string(cPolicyByte) + ann[BuildKey(pi.Name)] = string(cPolicyByte) // create update JSON patch jsonPatch, err := createReplaceJSONPatch(ann) return ann, jsonPatch, err diff --git a/pkg/annotations/controller.go b/pkg/annotations/controller.go index 222d91c688..b57077989f 100644 --- a/pkg/annotations/controller.go +++ b/pkg/annotations/controller.go @@ -79,7 +79,7 @@ func (pc *controller) handleErr(err error, key interface{}) { return } // This controller retries if something goes wrong. After that, it stops trying. - if pc.queue.NumRequeues(key) < WorkQueueRetryLimit { + if pc.queue.NumRequeues(key) < workQueueRetryLimit { glog.Warningf("Error syncing events %v: %v", key, err) // Re-enqueue the key rate limited. Based on the rate limiter on the // queue and the re-enqueue history, the key will be processed later again. diff --git a/pkg/annotations/info.go b/pkg/annotations/info.go new file mode 100644 index 0000000000..dbf38f4226 --- /dev/null +++ b/pkg/annotations/info.go @@ -0,0 +1,18 @@ +package annotations + +type info struct { + RKind string + RNs string + RName string + //TODO:Hack as slice makes the struct unhasable + Patch *[]byte +} + +func newInfo(rkind, rns, rname string, patch []byte) info { + return info{ + RKind: rkind, + RNs: rns, + RName: rname, + Patch: &patch, + } +} diff --git a/pkg/annotations/utils.go b/pkg/annotations/utils.go index 7ee8e69ba4..2c30938d7c 100644 --- a/pkg/annotations/utils.go +++ b/pkg/annotations/utils.go @@ -2,21 +2,15 @@ package annotations const annotationQueueName = "annotation-queue" const workerThreadCount = 1 -const WorkQueueRetryLimit = 3 +const workQueueRetryLimit = 3 -type info struct { - RKind string - RNs string - RName string - //TODO:Hack as slice makes the struct unhasable - Patch *[]byte -} - -func newInfo(rkind, rns, rname string, patch []byte) info { - return info{ - RKind: rkind, - RNs: rns, - RName: rname, - Patch: &patch, +func getStatus(status bool) string { + if status { + return "Success" } + return "Failure" +} + +func BuildKey(policyName string) string { + return "policies.kyverno.io." + policyName } diff --git a/pkg/apis/policy/v1alpha1/types.go b/pkg/apis/policy/v1alpha1/types.go index 50be3f5b8e..9842d2c929 100644 --- a/pkg/apis/policy/v1alpha1/types.go +++ b/pkg/apis/policy/v1alpha1/types.go @@ -93,8 +93,9 @@ type Violation struct { // FailedRule stored info and type of failed rules type FailedRule struct { - Name string `json:"name"` - Type string `json:"type"` //Mutation, Validation, Genertaion + Name string `json:"name"` + Type string `json:"type"` //Mutation, Validation, Genertaion + Error string `json:"error"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/controller/cleanup.go b/pkg/controller/cleanup.go index 486bcdc783..9d42fbc136 100644 --- a/pkg/controller/cleanup.go +++ b/pkg/controller/cleanup.go @@ -28,7 +28,6 @@ func cleanAnnotations(client *client.Client, obj interface{}) { for _, rule := range policy.Spec.Rules { for _, k := range rule.Kinds { if k == "Namespace" { - // REWORK: will be handeled by namespace controller continue } // kind -> resource @@ -63,7 +62,7 @@ func cleanAnnotations(client *client.Client, obj interface{}) { // get annotations ann := obj.GetAnnotations() - _, patch, err := annotations.RemovePolicyJSONPatch(ann, policy.Name) + _, patch, err := annotations.RemovePolicyJSONPatch(ann, annotations.BuildKey(policy.Name)) if err != nil { glog.Error(err) continue diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index dc7a1ee8df..272086f972 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -87,7 +87,6 @@ func (pc *PolicyController) deletePolicyHandler(resource interface{}) { glog.Error("error decoding object, invalid type") return } - //TODO: need to clear annotations on the resources cleanAnnotations(pc.client, resource) glog.Infof("policy deleted: %s", object.GetName()) } @@ -176,8 +175,7 @@ func (pc *PolicyController) syncHandler(obj interface{}) error { glog.Errorf("invalid policy key: %s", key) return nil } - - // Get Policy resource with namespace/name + // Get Policy policy, err := pc.policyLister.Get(name) if err != nil { if errors.IsNotFound(err) { @@ -186,20 +184,20 @@ func (pc *PolicyController) syncHandler(obj interface{}) error { } return err } - // process policy on existing resource - // get the violations and pass to violation Builder - // get the events and pass to event Builder - //TODO: processPolicy + glog.Infof("process policy %s on existing resources", policy.GetName()) + // Process policy on existing resources policyInfos := engine.ProcessExisting(pc.client, policy) + events, violations := pc.createEventsAndViolations(policyInfos) + // Events, Violations pc.eventController.Add(events...) err = pc.violationBuilder.Add(violations...) if err != nil { glog.Error(err) } - // add annotations to resources + // Annotations pc.createAnnotations(policyInfos) return nil @@ -277,9 +275,11 @@ func (pc *PolicyController) createEventsAndViolations(policyInfos []*info.Policy case info.Generation: frule.Type = info.Generation.String() } + frule.Error = rule.GetErrorString() default: glog.Info("Unsupported Rule type") } + frule.Error = rule.GetErrorString() frules = append(frules, frule) events = append(events, e) } else { @@ -291,24 +291,13 @@ func (pc *PolicyController) createEventsAndViolations(policyInfos []*info.Policy e := event.NewEvent("Policy", "", policyInfo.Name, event.PolicyViolation, event.FResourcePolcy, policyInfo.RNamespace+"/"+policyInfo.RName, concatFailedRules(frules)) events = append(events, e) // Violation - // TODO: Violation is currently create at policy, level not resource level - // As we create violation, we check if the - v := violation.BuldNewViolation(policyInfo.Name, policyInfo.RKind, policyInfo.RNamespace, policyInfo.RName, event.PolicyViolation.String(), frules) + v := violation.BuldNewViolation(policyInfo.Name, policyInfo.RKind, policyInfo.RNamespace, policyInfo.RName, event.PolicyViolation.String(), policyInfo.GetFailedRules()) violations = append(violations, v) } else { // clean up violations pc.violationBuilder.RemoveInactiveViolation(policyInfo.Name, policyInfo.RKind, policyInfo.RNamespace, policyInfo.RName, info.Mutation) pc.violationBuilder.RemoveInactiveViolation(policyInfo.Name, policyInfo.RKind, policyInfo.RNamespace, policyInfo.RName, info.Validation) } - - // else { - // // Policy was processed succesfully - // e := event.NewEvent("Policy", "", policyInfo.Name, event.PolicyApplied, event.SPolicyApply, policyInfo.Name) - // events = append(events, e) - // // Policy applied succesfully on resource - // e = event.NewEvent(policyInfo.RKind, policyInfo.RNamespace, policyInfo.RName, event.PolicyApplied, event.SRuleApply, strings.Join(sruleNames, ";"), policyInfo.RName) - // events = append(events, e) - // } } return events, violations } diff --git a/pkg/gencontroller/generation.go b/pkg/gencontroller/generation.go index f153d2434a..c86b408294 100644 --- a/pkg/gencontroller/generation.go +++ b/pkg/gencontroller/generation.go @@ -78,9 +78,7 @@ func (c *Controller) processPolicy(ns *corev1.Namespace, p *v1alpha1.Policy) { if onViolation { glog.Infof("Adding violation for generation rule of policy %s\n", policyInfo.Name) - - v := violation.NewViolation(event.PolicyViolation, policyInfo.Name, policyInfo.RKind, policyInfo.RName, - policyInfo.RNamespace, msg) + v := violation.BuldNewViolation(policyInfo.Name, policyInfo.RKind, policyInfo.RNamespace, policyInfo.RName, event.PolicyViolation.String(), policyInfo.GetFailedRules()) c.violationBuilder.Add(v) } else { eventInfo = event.NewEvent(policyKind, "", policyInfo.Name, event.RequestBlocked, diff --git a/pkg/info/info.go b/pkg/info/info.go index 260dbb35e0..d87ce0d7b2 100644 --- a/pkg/info/info.go +++ b/pkg/info/info.go @@ -69,7 +69,7 @@ func (pi *PolicyInfo) GetFailedRules() []v1alpha1.FailedRule { var rules []v1alpha1.FailedRule for _, r := range pi.Rules { if !r.IsSuccessful() { - rules = append(rules, v1alpha1.FailedRule{Name: r.Name, Type: r.RuleType.String()}) + rules = append(rules, v1alpha1.FailedRule{Name: r.Name, Type: r.RuleType.String(), Error: r.GetErrorString()}) } } return rules @@ -111,12 +111,18 @@ type RuleInfo struct { } //ToString reule information +//TODO: check if this is needed func (ri *RuleInfo) ToString() string { str := "rulename: " + ri.Name msgs := strings.Join(ri.Msgs, ";") return strings.Join([]string{str, msgs}, ";") } +//GetErrorString returns the error message for a rule +func (ri *RuleInfo) GetErrorString() string { + return strings.Join(ri.Msgs, ";") +} + //NewRuleInfo creates a new RuleInfo func NewRuleInfo(ruleName string, ruleType RuleType) *RuleInfo { return &RuleInfo{ diff --git a/pkg/violation/builder.go b/pkg/violation/builder.go index 75514e35ec..6f8f68aa7a 100644 --- a/pkg/violation/builder.go +++ b/pkg/violation/builder.go @@ -2,10 +2,7 @@ package violation import ( "errors" - "fmt" - "reflect" - "github.com/golang/glog" v1alpha1 "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" lister "github.com/nirmata/kyverno/pkg/client/listers/policy/v1alpha1" client "github.com/nirmata/kyverno/pkg/dclient" @@ -32,7 +29,6 @@ type builder struct { type Builder interface { Generator processViolation(info *Info) error - isActive(kind string, rname string, rnamespace string) (bool, error) } //NewPolicyViolationBuilder returns new violation builder @@ -48,6 +44,20 @@ func NewPolicyViolationBuilder(client *client.Client, return builder } +//BuldNewViolation returns a new violation +func BuldNewViolation(pName string, rKind string, rNs string, rName string, reason string, frules []v1alpha1.FailedRule) *Info { + return &Info{ + Policy: pName, + Violation: v1alpha1.Violation{ + Kind: rKind, + Namespace: rNs, + Name: rName, + Reason: reason, + Rules: frules, + }, + } +} + func (b *builder) Add(infos ...*Info) error { if infos == nil { return nil @@ -131,6 +141,7 @@ func (b *builder) processViolation(info *Info) error { return nil } +//RemoveInactiveViolation func (b *builder) RemoveInactiveViolation(policy, rKind, rNs, rName string, ruleType info.RuleType) error { statusMap := map[string]interface{}{} violationsMap := map[string]interface{}{} @@ -204,6 +215,7 @@ func (b *builder) RemoveInactiveViolation(policy, rKind, rNs, rName string, rule return nil } +// ResourceRemoval on resources reoval we remove the policy violation in the policy func (b *builder) ResourceRemoval(policy, rKind, rNs, rName string) error { statusMap := map[string]interface{}{} violationsMap := map[string]interface{}{} @@ -257,325 +269,3 @@ func (b *builder) ResourceRemoval(policy, rKind, rNs, rName string) error { } return nil } - -// func (b *builder) processViolation(info *Info) error { -// currVs := map[string]interface{}{} -// statusMap := map[string]interface{}{} -// var ok bool -// //TODO: hack get from client -// p1, err := b.client.GetResource("Policy", "", info.Policy, "status") -// if err != nil { -// return err -// } -// unstr := p1.UnstructuredContent() -// // check if "status" field exists -// status, ok := unstr["status"] -// if ok { -// // status is already present then we append violations -// if statusMap, ok = status.(map[string]interface{}); !ok { -// return errors.New("Unable to parse status subresource") -// } -// violations, ok := statusMap["violations"] -// if !ok { -// glog.Info("violation not present") -// } -// // Violations map[string][]Violations -// glog.Info(reflect.TypeOf(violations)) -// if currVs, ok = violations.(map[string]interface{}); !ok { -// return errors.New("Unable to parse violations") -// } -// } -// // Info: -// // Key - Kind, Namespace, Name -// // policy - Name -// // violation, ok := currVs[info.getKey()] -// // Key -> resource -// // 1> Check if there were any previous violations for the given key -// // 2> If No, create a new one -// if !ok { -// currVs[info.getKey()] = info.Violation -// } else { -// currV := currVs[info.getKey()] -// glog.Info(reflect.TypeOf(currV)) -// v, ok := currV.(map[string]interface{}) -// if !ok { -// glog.Info("type not matching") -// } -// // get rules -// rules, ok := v["rules"] -// if !ok { -// glog.Info("rules not found") -// } -// glog.Info(reflect.TypeOf(rules)) -// rs, ok := rules.([]interface{}) -// if !ok { -// glog.Info("type not matching") -// } -// // check if rules are samre -// if isRuleNamesEqual(rs, info.Violation.Rules) { -// return nil -// } -// // else update the errors -// currVs[info.getKey()] = info.Violation -// } -// // newViolation := info.Violation -// // for _, violation := range currentViolations { -// // glog.Info(reflect.TypeOf(violation)) -// // if v, ok := violation.(map[string]interface{}); ok { -// // if name, ok := v["name"].(string); ok { -// // if namespace, ok := v["namespace"].(string); ok { -// // ok, err := b.isActive(info.Kind, name, namespace) -// // if err != nil { -// // glog.Error(err) -// // continue -// // } -// // if !ok { -// // //TODO remove the violation as it corresponds to resource that does not exist -// // glog.Info("removed violation") -// // } -// // } -// // } -// // } -// // } -// // currentViolations = append(currentViolations, newViolation) -// // // update violations -// // set the updated status -// statusMap["violations"] = currVs -// unstr["status"] = statusMap -// p1.SetUnstructuredContent(unstr) -// _, err = b.client.UpdateStatusResource("Policy", "", p1, false) -// if err != nil { -// return err -// } -// return nil -// } - -func (b *builder) isActive(kind, rname, rnamespace string) (bool, error) { - // Generate Merge Patch - _, err := b.client.GetResource(b.client.DiscoveryClient.GetGVRFromKind(kind).Resource, rnamespace, rname) - if err != nil { - glog.Errorf("unable to get resource %s/%s ", rnamespace, rname) - return false, err - } - return true, nil -} - -//NewViolation return new policy violation -func NewViolation(reason event.Reason, policyName, kind, rname, rnamespace, msg string) *Info { - return &Info{Policy: policyName, - Violation: v1alpha1.Violation{ - Kind: kind, - Name: rname, - Namespace: rnamespace, - Reason: reason.String(), - }, - } -} - -// //NewViolationFromEvent returns violation info from event -// func NewViolationFromEvent(e *event.Info, pName, rKind, rName, rnamespace string) *Info { -// return &Info{ -// Policy: pName, -// Violation: types.Violation{ -// Kind: rKind, -// Name: rName, -// Namespace: rnamespace, -// Reason: e.Reason, -// Message: e.Message, -// }, -// } -// } -// Build a new Violation -func BuldNewViolation(pName string, rKind string, rNs string, rName string, reason string, frules []v1alpha1.FailedRule) *Info { - return &Info{ - Policy: pName, - Violation: v1alpha1.Violation{ - Kind: rKind, - Namespace: rNs, - Name: rName, - Reason: reason, - Rules: frules, - }, - } -} - -func removeRuleTypes(currRules []interface{}, ruleType info.RuleType) ([]interface{}, error) { - //rules := []v1alpha1.FailedRule{} - var rules []interface{} - // removedRuleCount := 0 - for _, r := range currRules { - glog.Info(reflect.TypeOf(r)) - rfule, ok := r.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("incorrect type") - } - glog.Info(reflect.TypeOf(rfule["type"])) - rtype, ok := rfule["type"].(string) - if !ok { - return nil, fmt.Errorf("incorrect type") - } - name, ok := rfule["name"].(string) - if !ok { - return nil, fmt.Errorf("incorrect type") - } - if rtype != ruleType.String() { - rules = append(rules, v1alpha1.FailedRule{Name: name, Type: rtype}) - } - } - return rules, nil -} - -func isRuleNamesEqual(currRules []interface{}, newRules []v1alpha1.FailedRule) bool { - if len(currRules) != len(newRules) { - return false - } - for i, r := range currRules { - glog.Info(reflect.TypeOf(r)) - rfule, ok := r.(map[string]interface{}) - if !ok { - return false - } - glog.Info(reflect.TypeOf(rfule["name"])) - // name - name, ok := rfule["name"].(string) - if !ok { - return false - } - if name != newRules[i].Name { - return false - } - // type - - rtype, ok := rfule["type"].(string) - if !ok { - return false - } - if rtype != newRules[i].Type { - return false - } - - } - return true -} - -// //RemoveViolation will remove the violation for the resource if there was one -// func (b *builder) RemoveInactiveViolation(policy, rKind, rNs, rName string, ruleType info.RuleType) error { -// // Remove the pair from map -// statusMap := map[string]interface{}{} -// currVs := map[string]interface{}{} -// // Get the policy -// p, err := b.client.GetResource("Policy", "", policy, "status") -// if err != nil { -// glog.Infof("policy %s not found", policy) -// return err -// } -// unstr := p.UnstructuredContent() - -// // check if "status" field exists -// status, ok := unstr["status"] -// if ok { -// // status is already present then we append violations -// if statusMap, ok = status.(map[string]interface{}); !ok { -// return errors.New("Unable to parse status subresource") -// } -// violations, ok := statusMap["violations"] -// if !ok { -// glog.Info("violation not present") -// } -// glog.Info(reflect.TypeOf(violations)) -// if currVs, ok = violations.(map[string]interface{}); !ok { -// return errors.New("Unable to parse violations") -// } -// currV, ok := currVs[BuildKey(rKind, rNs, rName)] -// if !ok { -// // No Violation present -// return nil -// } -// glog.Info(reflect.TypeOf(currV)) -// v, ok := currV.(map[string]interface{}) -// if !ok { -// glog.Info("type not matching") -// } -// // get rules -// rules, ok := v["rules"] -// if !ok { -// glog.Info("rules not found") -// } -// glog.Info(reflect.TypeOf(rules)) -// rs, ok := rules.([]interface{}) -// if !ok { -// glog.Info("type not matching") -// } -// // Remove rules of defined type -// newrs, err := removeRuleTypes(rs, ruleType) -// if err != nil { -// glog.Info(err) -// } -// if newrs == nil { -// // all records are removed and is empty -// glog.Info("can remove the record") -// delete(currVs, BuildKey(rKind, rNs, rName)) -// } else { -// v["rules"] = newrs -// // update the violation with new rule -// currVs[BuildKey(rKind, rNs, rName)] = v -// } -// // update violations -// statusMap["violations"] = currVs -// // update status -// unstr["status"] = statusMap -// p.SetUnstructuredContent(unstr) -// _, err = b.client.UpdateStatusResource("Policy", "", p, false) -// if err != nil { -// return err -// } -// } -// return nil -// } - -// func (b *builder) ResourceRemoval(policy, rKind, rNs, rName string) error { -// // Remove the pair from map -// statusMap := map[string]interface{}{} -// currVs := map[string]interface{}{} -// // Get the policy -// p, err := b.client.GetResource("Policy", "", policy, "status") -// if err != nil { -// glog.Infof("policy %s not found", policy) -// return err -// } -// unstr := p.UnstructuredContent() -// // check if "status" field exists -// status, ok := unstr["status"] -// if ok { -// // status is already present then we append violations -// if statusMap, ok = status.(map[string]interface{}); !ok { -// return errors.New("Unable to parse status subresource") -// } -// violations, ok := statusMap["violations"] -// if !ok { -// glog.Info("violation not present") -// } -// glog.Info(reflect.TypeOf(violations)) -// if currVs, ok = violations.(map[string]interface{}); !ok { -// return errors.New("Unable to parse violations") -// } -// _, ok = currVs[BuildKey(rKind, rNs, rName)] -// if !ok { -// // No Violation for this resource -// return nil -// } -// // remove the pair from the map -// delete(currVs, BuildKey(rKind, rNs, rName)) -// glog.Info("Removed Violation") -// // update violations -// statusMap["violations"] = currVs -// // update status -// unstr["status"] = statusMap -// p.SetUnstructuredContent(unstr) -// _, err = b.client.UpdateStatusResource("Policy", "", p, false) -// if err != nil { -// return err -// } -// } -// return nil -// } diff --git a/pkg/violation/util.go b/pkg/violation/util.go index bd0b9a1c3a..3f525b53d2 100644 --- a/pkg/violation/util.go +++ b/pkg/violation/util.go @@ -11,7 +11,7 @@ const workqueueViolationName = "Policy-Violations" // Event Reason const violationEventResrouce = "Violation" -//ViolationInfo describes the policyviolation details +//Info describes the policyviolation details type Info struct { Policy string policytype.Violation @@ -25,4 +25,3 @@ func (i Info) getKey() string { func BuildKey(rKind, rNs, rName string) string { return rKind + "/" + rNs + "/" + rName } - diff --git a/pkg/webhooks/validation.go b/pkg/webhooks/validation.go index b8d5eaa91d..51e9dc6071 100644 --- a/pkg/webhooks/validation.go +++ b/pkg/webhooks/validation.go @@ -14,8 +14,6 @@ import ( // If there are no errors in validating rule we apply generation rules func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { policyInfos := []*info.PolicyInfo{} - // var violations []*violation.Info - // var eventsInfo []*event.Info policies, err := ws.policyLister.List(labels.NewSelector()) if err != nil { // Unable to connect to policy Lister to access policies @@ -81,6 +79,7 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 } } policyInfos = append(policyInfos, policyInfo) + // annotations annPatch := addAnnotationsToResource(request.Object.Raw, policyInfo, info.Validation) if annPatch != nil { if annPatches == nil { From 91030987ea920a6fb12f3a593760f237f4aa3314 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Fri, 19 Jul 2019 16:17:10 -0700 Subject: [PATCH 18/24] handle retrys events --- main.go | 8 ++++---- pkg/controller/utils.go | 2 +- pkg/event/controller.go | 35 ++++++++++++++++++++++++++--------- pkg/event/util.go | 2 ++ 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/main.go b/main.go index 116bbb5331..d3fb1a6a8f 100644 --- a/main.go +++ b/main.go @@ -68,6 +68,10 @@ func main() { stopCh := signals.SetupSignalHandler() + if err = webhookRegistrationClient.Register(); err != nil { + glog.Fatalf("Failed registering Admission Webhooks: %v\n", err) + } + policyInformerFactory.Run(stopCh) kubeInformer.Start(stopCh) eventController.Run(stopCh) @@ -77,10 +81,6 @@ func main() { glog.Fatalf("Error running PolicyController: %v\n", err) } - if err = webhookRegistrationClient.Register(); err != nil { - glog.Fatalf("Failed registering Admission Webhooks: %v\n", err) - } - server.RunAsync() <-stopCh server.Stop() diff --git a/pkg/controller/utils.go b/pkg/controller/utils.go index f353d1408c..2501addb51 100644 --- a/pkg/controller/utils.go +++ b/pkg/controller/utils.go @@ -8,7 +8,7 @@ import ( const policyWorkQueueName = "policyworkqueue" -const policyWorkQueueRetryLimit = 5 +const policyWorkQueueRetryLimit = 3 const policyControllerWorkerCount = 2 diff --git a/pkg/event/controller.go b/pkg/event/controller.go index 819f97cf73..ef359a4d0b 100644 --- a/pkg/event/controller.go +++ b/pkg/event/controller.go @@ -1,7 +1,6 @@ package event import ( - "fmt" "time" "github.com/golang/glog" @@ -92,40 +91,58 @@ func (c *controller) Stop() { defer c.queue.ShutDown() glog.Info("Shutting down eventbuilder controller workers") } + func (c *controller) runWorker() { for c.processNextWorkItem() { } } +func (c *controller) handleErr(err error, key interface{}) { + if err == nil { + c.queue.Forget(key) + return + } + // This controller retries if something goes wrong. After that, it stops trying. + if c.queue.NumRequeues(key) < WorkQueueRetryLimit { + glog.Warningf("Error syncing events %v: %v", key, err) + // Re-enqueue the key rate limited. Based on the rate limiter on the + // queue and the re-enqueue history, the key will be processed later again. + c.queue.AddRateLimited(key) + return + } + c.queue.Forget(key) + glog.Error(err) + glog.Warningf("Dropping the key out of the queue: %v", err) +} + func (c *controller) processNextWorkItem() bool { obj, shutdown := c.queue.Get() if shutdown { return false } + err := func(obj interface{}) error { defer c.queue.Done(obj) var key Info var ok bool + if key, ok = obj.(Info); !ok { c.queue.Forget(obj) glog.Warningf("Expecting type info by got %v\n", obj) return nil } - // Run the syncHandler, passing the resource and the policy - if err := c.SyncHandler(key); err != nil { - c.queue.AddRateLimited(key) - return fmt.Errorf("error syncing '%s' : %s, requeuing event creation request", key.Namespace+"/"+key.Name, err.Error()) - } + err := c.syncHandler(key) + c.handleErr(err, obj) return nil }(obj) - if err != nil { - glog.Warning(err) + glog.Error(err) + return true } return true } -func (c *controller) SyncHandler(key Info) error { +func (c *controller) syncHandler(key Info) error { var robj runtime.Object var err error diff --git a/pkg/event/util.go b/pkg/event/util.go index f3601d05fd..fb1f97007e 100644 --- a/pkg/event/util.go +++ b/pkg/event/util.go @@ -6,6 +6,8 @@ const eventWorkQueueName = "policy-controller-events" const eventWorkerThreadCount = 1 +const WorkQueueRetryLimit = 1 + //Info defines the event details type Info struct { Kind string From 4166d136844c2d127997e0820da0e8e7187b4f9d Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Fri, 19 Jul 2019 16:18:36 -0700 Subject: [PATCH 19/24] correct case --- pkg/event/controller.go | 2 +- pkg/event/util.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/event/controller.go b/pkg/event/controller.go index ef359a4d0b..5c5e9fca0b 100644 --- a/pkg/event/controller.go +++ b/pkg/event/controller.go @@ -103,7 +103,7 @@ func (c *controller) handleErr(err error, key interface{}) { return } // This controller retries if something goes wrong. After that, it stops trying. - if c.queue.NumRequeues(key) < WorkQueueRetryLimit { + if c.queue.NumRequeues(key) < workQueueRetryLimit { glog.Warningf("Error syncing events %v: %v", key, err) // Re-enqueue the key rate limited. Based on the rate limiter on the // queue and the re-enqueue history, the key will be processed later again. diff --git a/pkg/event/util.go b/pkg/event/util.go index fb1f97007e..34daa0a9b1 100644 --- a/pkg/event/util.go +++ b/pkg/event/util.go @@ -6,7 +6,7 @@ const eventWorkQueueName = "policy-controller-events" const eventWorkerThreadCount = 1 -const WorkQueueRetryLimit = 1 +const workQueueRetryLimit = 1 //Info defines the event details type Info struct { From 62b0f5703a4405846f52f5eb98b8dd718a014bae Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Fri, 19 Jul 2019 16:22:26 -0700 Subject: [PATCH 20/24] add error msg only for failed rules --- pkg/annotations/annotations.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 92623805a3..2b05aef6f7 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -36,8 +36,12 @@ func getRules(rules []*pinfo.RuleInfo, ruleType pinfo.RuleType) map[string]Rule if r.RuleType != ruleType { continue } - annrules[r.Name] = - Rule{Status: getStatus(r.IsSuccessful()), Error: r.GetErrorString()} + + rule := Rule{Status: getStatus(r.IsSuccessful())} + if !r.IsSuccessful() { + rule.Error = r.GetErrorString() + } + annrules[r.Name] = rule } return annrules } From 9fd59297f8d0ec63669bc57f694b5cdbbb455ecf Mon Sep 17 00:00:00 2001 From: Shuting Zhao Date: Fri, 19 Jul 2019 17:52:24 -0700 Subject: [PATCH 21/24] remove rule name in failure even info --- pkg/engine/mutation.go | 6 +++--- pkg/engine/validation.go | 2 +- pkg/webhooks/mutation.go | 2 +- pkg/webhooks/validation.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/engine/mutation.go b/pkg/engine/mutation.go index 6d5b2b0b32..6c6b9b680b 100644 --- a/pkg/engine/mutation.go +++ b/pkg/engine/mutation.go @@ -29,7 +29,7 @@ func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersio overlayPatches, err := ProcessOverlay(rule, rawResource, gvk) if err != nil { ri.Fail() - ri.Addf("Rule %s: Overlay application has failed, err %s.", rule.Name, err) + ri.Addf("overlay application has failed, err %v.", err) } else { // Apply the JSON patches from the rule to the resource rawResource, err = ApplyPatches(rawResource, overlayPatches) @@ -49,14 +49,14 @@ func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersio if len(errs) > 0 { ri.Fail() for _, err := range errs { - ri.Addf("Rule %s: Patches application has failed, err %s.", rule.Name, err) + ri.Addf("patches application has failed, err %v.", err) } } else { // Apply the JSON patches from the rule to the resource rawResource, err = ApplyPatches(rawResource, rulePatches) if err != nil { ri.Fail() - ri.Addf("Unable to apply JSON patch to resource, err %s.", err) + ri.Addf("Unable to apply JSON patch to resource, err %v.", err) } else { ri.Addf("Rule %s: Patches succesfully applied.", rule.Name) allPatches = append(allPatches, rulePatches...) diff --git a/pkg/engine/validation.go b/pkg/engine/validation.go index 8210f6e609..832feffb35 100644 --- a/pkg/engine/validation.go +++ b/pkg/engine/validation.go @@ -41,7 +41,7 @@ func Validate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVers err := validateResourceWithPattern(resource, rule.Validation.Pattern) if err != nil { ri.Fail() - ri.Addf("Rule %s: Validation has failed, err %s.", rule.Name, err) + ri.Addf("validation has failed, err %v.", err) } else { ri.Addf("Rule %s: Validation succesfully.", rule.Name) diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go index 9dd39b6bf0..f1e789edf2 100644 --- a/pkg/webhooks/mutation.go +++ b/pkg/webhooks/mutation.go @@ -59,7 +59,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be if !policyInfo.IsSuccessful() { glog.Infof("Failed to apply policy %s on resource %s/%s", policy.Name, rname, rns) for _, r := range ruleInfos { - glog.Warning(r.Msgs) + glog.Warningf("%s: %s\n", r.Name, r.Msgs) } } else { // // CleanUp Violations if exists diff --git a/pkg/webhooks/validation.go b/pkg/webhooks/validation.go index 51e9dc6071..5c90c92c12 100644 --- a/pkg/webhooks/validation.go +++ b/pkg/webhooks/validation.go @@ -65,7 +65,7 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1 if !policyInfo.IsSuccessful() { glog.Infof("Failed to apply policy %s on resource %s/%s", policy.Name, rname, rns) for _, r := range ruleInfos { - glog.Warning(r.Msgs) + glog.Warningf("%s: %s\n", r.Name, r.Msgs) } } else { // CleanUp Violations if exists From 3cb978c16fba401e9b9b91c3daa6b24d07a2bb9f Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Fri, 19 Jul 2019 20:30:55 -0700 Subject: [PATCH 22/24] clean up + fix bugs --- definitions/install.yaml | 1 - definitions/install_debug.yaml | 11 ++-- examples/cli/nginx.yaml | 2 +- examples/cli/policy_deployment.yaml | 1 - examples/demo/mutate_patch/policy_patch.yaml | 1 + examples/demo/qos/policy_qos.yaml | 63 ++++++++++--------- examples/demo/qos/qos.yaml | 8 +-- examples/mutate/overlay/nginx.yaml | 2 +- .../overlay/policy_imagePullPolicy.yaml | 2 +- examples/mutate/patches/endpoints.yaml | 2 +- pkg/annotations/annotations.go | 34 ++++++++-- pkg/annotations/annotations_test.go | 13 ++-- pkg/controller/controller.go | 2 +- pkg/controller/controller_test.go | 3 +- pkg/engine/mutation.go | 30 +++------ pkg/info/info.go | 9 ++- pkg/testrunner/test.go | 2 +- pkg/webhooks/mutation.go | 34 +++++----- pkg/webhooks/report.go | 2 - pkg/webhooks/utils.go | 1 - test/StatefulSet/policy-StatefulSet.yaml | 2 +- 21 files changed, 115 insertions(+), 110 deletions(-) diff --git a/definitions/install.yaml b/definitions/install.yaml index 22c7806ba4..7b44e59dc8 100644 --- a/definitions/install.yaml +++ b/definitions/install.yaml @@ -184,7 +184,6 @@ spec: containers: - name: kyverno image: nirmata/kyverno:latest -# image: registry-v2.nirmata.io/nirmata/kyverno:testImage args: ["--filterKind","Nodes,Events,APIService,SubjectAccessReview"] ports: - containerPort: 443 diff --git a/definitions/install_debug.yaml b/definitions/install_debug.yaml index f136f80978..732b3cd5f7 100644 --- a/definitions/install_debug.yaml +++ b/definitions/install_debug.yaml @@ -21,16 +21,13 @@ spec: spec: required: - rules - # set as required, as we cannot set default yet. check below for more details - - mode properties: - mode: + # default values to be handled by user + validationFailureAction: type: string - # default can only be set if CustomResourceDefaulting feature gate is enabled - # default: blockChanges enum: - - blockChanges - - reportViolation + - block + - report rules: type: array items: diff --git a/examples/cli/nginx.yaml b/examples/cli/nginx.yaml index db1db6b186..cc6d30eead 100644 --- a/examples/cli/nginx.yaml +++ b/examples/cli/nginx.yaml @@ -20,4 +20,4 @@ spec: image: nginx:1.7.9 imagePullPolicy: Always ports: - - containerPort: 80 + - containerPort: 80 \ No newline at end of file diff --git a/examples/cli/policy_deployment.yaml b/examples/cli/policy_deployment.yaml index 5957912c55..064b68bb4d 100644 --- a/examples/cli/policy_deployment.yaml +++ b/examples/cli/policy_deployment.yaml @@ -35,4 +35,3 @@ spec : containers: - (image): "nginx*" imagePullPolicy: Always - diff --git a/examples/demo/mutate_patch/policy_patch.yaml b/examples/demo/mutate_patch/policy_patch.yaml index c67f9ef752..6ef8e9f0c7 100644 --- a/examples/demo/mutate_patch/policy_patch.yaml +++ b/examples/demo/mutate_patch/policy_patch.yaml @@ -11,6 +11,7 @@ spec : selector: matchLabels: label : test + name: demo-endpoint mutate: patches: # add a new label diff --git a/examples/demo/qos/policy_qos.yaml b/examples/demo/qos/policy_qos.yaml index 49aabe72bd..3e37a83dae 100644 --- a/examples/demo/qos/policy_qos.yaml +++ b/examples/demo/qos/policy_qos.yaml @@ -3,38 +3,39 @@ kind: Policy metadata: name: policy-qos spec: + validationFailureAction: "report" rules: - - name: add-memory-limit - resource: - kinds: - - Deployment - mutate: - overlay: - spec: - template: - spec: - containers: - # the wildcard * will match all containers in the list + - name: add-memory-limit + resource: + kinds: + - Deployment + mutate: + overlay: + spec: + template: + spec: + containers: + # the wildcard * will match all containers in the list + - (name): "*" + resources: + limits: + # add memory limit if it is not exist + "+(memory)": "300Mi" + - name: check-cpu-memory-limits + resource: + kinds: + - Deployment + validate: + message: "Resource limits are required for CPU and memory" + pattern: + spec: + template: + spec: + containers: + # match all contianers - (name): "*" resources: limits: - # add memory limit if it is not exist - "+(memory)": "300Mi" - - name: check-cpu-memory-limits - resource: - kinds: - - Deployment - validate: - message: "Resource limits are required for CPU and memory" - pattern: - spec: - template: - spec: - containers: - # match all contianers - - (name): "*" - resources: - limits: - # cpu and memory are required - memory: "?*" - cpu: "?*" \ No newline at end of file + # cpu and memory are required + memory: "?*" + cpu: "?*" diff --git a/examples/demo/qos/qos.yaml b/examples/demo/qos/qos.yaml index 0b8018b673..c81b65b0e5 100644 --- a/examples/demo/qos/qos.yaml +++ b/examples/demo/qos/qos.yaml @@ -17,10 +17,4 @@ spec: image: nginx:latest resources: limits: - cpu: "50m" - - name: ghost - image: ghost:latest - resources: - limits: - cpu: "50m" - memory: "500Mi" + cpu: "50m" \ No newline at end of file diff --git a/examples/mutate/overlay/nginx.yaml b/examples/mutate/overlay/nginx.yaml index bdf22b13cb..107f97e446 100644 --- a/examples/mutate/overlay/nginx.yaml +++ b/examples/mutate/overlay/nginx.yaml @@ -22,4 +22,4 @@ spec: ports: - containerPort: 80 - name: ghost - image: ghost:latest + image: ghost:latest \ No newline at end of file diff --git a/examples/mutate/overlay/policy_imagePullPolicy.yaml b/examples/mutate/overlay/policy_imagePullPolicy.yaml index cb6ac8025f..46537f9cd7 100644 --- a/examples/mutate/overlay/policy_imagePullPolicy.yaml +++ b/examples/mutate/overlay/policy_imagePullPolicy.yaml @@ -16,4 +16,4 @@ spec: containers: # if the image tag is latest, set the imagePullPolicy to Always - (image): "*:latest" - imagePullPolicy: "IfNotPresent" + imagePullPolicy: "IfNotPresent" \ No newline at end of file diff --git a/examples/mutate/patches/endpoints.yaml b/examples/mutate/patches/endpoints.yaml index 958d931482..792a83da96 100644 --- a/examples/mutate/patches/endpoints.yaml +++ b/examples/mutate/patches/endpoints.yaml @@ -10,4 +10,4 @@ subsets: ports: - name: secure-connection port: 443 - protocol: TCP + protocol: TCP \ No newline at end of file diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 2b05aef6f7..e2dd9d2df3 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -26,6 +26,28 @@ type Rule struct { Error string `json:"error,omitempty"` } +func (p *Policy) getOverAllStatus() string { + // mutation + for _, v := range p.MutationRules { + if v.Status == "Failure" { + return "Failure" + } + } + // validation + for _, v := range p.ValidationRules { + if v.Status == "Failure" { + return "Failure" + } + } + // generation + for _, v := range p.GenerationRules { + if v.Status == "Failure" { + return "Failure" + } + } + return "Success" +} + func getRules(rules []*pinfo.RuleInfo, ruleType pinfo.RuleType) map[string]Rule { if len(rules) == 0 { return nil @@ -48,10 +70,6 @@ func getRules(rules []*pinfo.RuleInfo, ruleType pinfo.RuleType) map[string]Rule func (p *Policy) updatePolicy(obj *Policy, ruleType pinfo.RuleType) bool { updates := false - if p.Status != obj.Status { - updates = true - } - p.Status = obj.Status // Check Mutation rules switch ruleType { case pinfo.Mutation: @@ -66,7 +84,13 @@ func (p *Policy) updatePolicy(obj *Policy, ruleType pinfo.RuleType) bool { if p.compareGenerationRules(obj.GenerationRules) { updates = true } + if p.Status != obj.Status { + updates = true + } } + p.Status = obj.Status + // check if any rules failed + p.Status = p.getOverAllStatus() // If there are any updates then the annotation can be updated, can skip return updates } @@ -137,6 +161,7 @@ func AddPolicy(obj *unstructured.Unstructured, pi *pinfo.PolicyInfo, ruleType pi // 1> policy status // 2> Mutation, Validation, Generation if cPolicyObj.updatePolicy(PolicyObj, ruleType) { + cPolicyByte, err := json.Marshal(cPolicyObj) if err != nil { return false @@ -212,6 +237,7 @@ func AddPolicyJSONPatch(ann map[string]string, pi *pinfo.PolicyInfo, ruleType pi if !update { return nil, nil, err } + cPolicyByte, err := json.Marshal(cPolicyObj) if err != nil { return nil, nil, err diff --git a/pkg/annotations/annotations_test.go b/pkg/annotations/annotations_test.go index 58741ed824..558746a936 100644 --- a/pkg/annotations/annotations_test.go +++ b/pkg/annotations/annotations_test.go @@ -2,10 +2,9 @@ package annotations import ( "encoding/json" - "fmt" "testing" - "github.com/nirmata/kyverno/pkg/info" + pinfo "github.com/nirmata/kyverno/pkg/info" ) func TestAddPatch(t *testing.T) { @@ -13,27 +12,25 @@ func TestAddPatch(t *testing.T) { objRaw := []byte(`{"kind":"Deployment","apiVersion":"apps/v1","metadata":{"name":"nginx-deployment","namespace":"default","creationTimestamp":null,"labels":{"app":"nginx"}},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"nginx"}},"template":{"metadata":{"creationTimestamp":null,"labels":{"app":"nginx"}},"spec":{"containers":[{"name":"nginx","image":"nginx:latest","ports":[{"containerPort":80,"protocol":"TCP"}],"resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"},{"name":"ghost","image":"ghost:latest","resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","securityContext":{},"schedulerName":"default-scheduler"}},"strategy":{"type":"RollingUpdate","rollingUpdate":{"maxUnavailable":"25%","maxSurge":"25%"}},"revisionHistoryLimit":10,"progressDeadlineSeconds":600},"status":{}}`) piRaw := []byte(`{"Name":"set-image-pull-policy","RKind":"Deployment","RName":"nginx-deployment","RNamespace":"default","ValidationFailureAction":"","Rules":[{"Name":"nginx-deployment","Msgs":["Rule nginx-deployment: Overlay succesfully applied."],"RuleType":0}]}`) ann := ParseAnnotationsFromObject(objRaw) - pi := info.PolicyInfo{} + pi := pinfo.PolicyInfo{} err := json.Unmarshal(piRaw, &pi) if err != nil { panic(err) } - ann, patch, err := AddPolicyJSONPatch(ann, &pi, info.Mutation) + ann, _, err = AddPolicyJSONPatch(ann, &pi, pinfo.Mutation) if err != nil { panic(err) } - fmt.Println(string(patch)) // Update piRaw = []byte(`{"Name":"set-image-pull-policy","RKind":"Deployment","RName":"nginx-deployment","RNamespace":"default","ValidationFailureAction":"","Rules":[{"Name":"nginx-deployment","Msgs":["Rule nginx-deployment1: Overlay succesfully applied."],"RuleType":0}]}`) // ann = ParseAnnotationsFromObject(objRaw) - pi = info.PolicyInfo{} + pi = pinfo.PolicyInfo{} err = json.Unmarshal(piRaw, &pi) if err != nil { panic(err) } - ann, patch, err = AddPolicyJSONPatch(ann, &pi, info.Mutation) + ann, _, err = AddPolicyJSONPatch(ann, &pi, pinfo.Mutation) if err != nil { panic(err) } - fmt.Println(string(patch)) } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 272086f972..d66469c946 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -225,7 +225,7 @@ func (pc *PolicyController) createAnnotations(policyInfos []*info.PolicyInfo) { if err != nil { glog.Error(err) } - if mpatch == nil && mpatch == nil { + if mpatch == nil && vpatch == nil { //nothing to patch continue } diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 5df64bd924..34448ffb20 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -41,7 +41,8 @@ func (f *fixture) runControler(policyName string) { f.Client, policyInformerFactory, violationBuilder, - eventController) + eventController, + nil) stopCh := signals.SetupSignalHandler() // start informer & controller diff --git a/pkg/engine/mutation.go b/pkg/engine/mutation.go index 6d5b2b0b32..d4c974a0ee 100644 --- a/pkg/engine/mutation.go +++ b/pkg/engine/mutation.go @@ -10,7 +10,7 @@ import ( // Mutate performs mutation. Overlay first and then mutation patches func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersionKind) ([][]byte, []*info.RuleInfo) { var allPatches [][]byte - var err error + patchedDocument := rawResource ris := []*info.RuleInfo{} for _, rule := range policy.Spec.Rules { @@ -31,36 +31,26 @@ func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersio ri.Fail() ri.Addf("Rule %s: Overlay application has failed, err %s.", rule.Name, err) } else { - // Apply the JSON patches from the rule to the resource - rawResource, err = ApplyPatches(rawResource, overlayPatches) - if err != nil { - ri.Fail() - ri.Addf("Unable to apply JSON patch to resource, err %s.", err) - } else { - ri.Addf("Rule %s: Overlay succesfully applied.", rule.Name) - allPatches = append(allPatches, overlayPatches...) - } + ri.Addf("Rule %s: Overlay succesfully applied.", rule.Name) + //TODO: patchbytes -> string + //glog.V(3).Info(" Overlay succesfully applied. Patch %s", string(overlayPatches)) + allPatches = append(allPatches, overlayPatches...) } } // Process Patches if len(rule.Mutation.Patches) != 0 { - rulePatches, errs := ProcessPatches(rule, rawResource) + rulePatches, errs := ProcessPatches(rule, patchedDocument) if len(errs) > 0 { ri.Fail() for _, err := range errs { ri.Addf("Rule %s: Patches application has failed, err %s.", rule.Name, err) } } else { - // Apply the JSON patches from the rule to the resource - rawResource, err = ApplyPatches(rawResource, rulePatches) - if err != nil { - ri.Fail() - ri.Addf("Unable to apply JSON patch to resource, err %s.", err) - } else { - ri.Addf("Rule %s: Patches succesfully applied.", rule.Name) - allPatches = append(allPatches, rulePatches...) - } + ri.Addf("Rule %s: Patches succesfully applied.", rule.Name) + //TODO: patchbytes -> string + //glog.V(3).Info("Patches succesfully applied. Patch %s", string(overlayPatches)) + allPatches = append(allPatches, rulePatches...) } } ris = append(ris, ri) diff --git a/pkg/info/info.go b/pkg/info/info.go index d87ce0d7b2..a74458d1d2 100644 --- a/pkg/info/info.go +++ b/pkg/info/info.go @@ -39,7 +39,14 @@ func NewPolicyInfo(policyName, rKind, rName, rNamespace, validationFailureAction //IsSuccessful checks if policy is succesful // the policy is set to fail, if any of the rules have failed func (pi *PolicyInfo) IsSuccessful() bool { - return pi.success + for _, r := range pi.Rules { + if !r.success { + pi.success = false + return false + } + } + pi.success = true + return true } // SuccessfulRules returns list of successful rule names diff --git a/pkg/testrunner/test.go b/pkg/testrunner/test.go index d9963e7578..bc73f795b1 100644 --- a/pkg/testrunner/test.go +++ b/pkg/testrunner/test.go @@ -174,7 +174,7 @@ func (t *test) applyPolicy(policy *pt.Policy, rkind, rname, rns, - policy.Spec.Mode) + policy.Spec.ValidationFailureAction) // Apply Mutation Rules patches, ruleInfos := engine.Mutate(*policy, rawResource, *tresource.gvk) policyInfo.AddRuleInfos(ruleInfos) diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go index 9dd39b6bf0..ede3097738 100644 --- a/pkg/webhooks/mutation.go +++ b/pkg/webhooks/mutation.go @@ -2,7 +2,6 @@ package webhooks import ( jsonpatch "github.com/evanphx/json-patch" - "github.com/golang/glog" engine "github.com/nirmata/kyverno/pkg/engine" "github.com/nirmata/kyverno/pkg/info" @@ -23,6 +22,9 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be Allowed: true, } } + rname := engine.ParseNameFromObject(request.Object.Raw) + rns := engine.ParseNamespaceFromObject(request.Object.Raw) + rkind := engine.ParseKindFromObject(request.Object.Raw) var allPatches [][]byte var annPatches []byte @@ -38,9 +40,6 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be Allowed: true, } } - rname := engine.ParseNameFromObject(request.Object.Raw) - rns := engine.ParseNamespaceFromObject(request.Object.Raw) - rkind := engine.ParseKindFromObject(request.Object.Raw) policyInfo := info.NewPolicyInfo(policy.Name, rkind, rname, @@ -62,18 +61,17 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be glog.Warning(r.Msgs) } } else { - // // CleanUp Violations if exists - // err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Mutation) + // TODO + // // CleanUp Violations if exists + // err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Validation) // if err != nil { // glog.Info(err) // } - - if len(policyPatches) > 0 { - allPatches = append(allPatches, policyPatches...) - glog.Infof("Mutation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) - } + allPatches = append(allPatches, policyPatches...) + glog.Infof("Mutation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns) } policyInfos = append(policyInfos, policyInfo) + annPatch := addAnnotationsToResource(request.Object.Raw, policyInfo, info.Mutation) if annPatch != nil { if annPatches == nil { @@ -91,20 +89,18 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be eventsInfo, _ := newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update), info.Mutation) ws.eventController.Add(eventsInfo...) } + // add annotations + if annPatches != nil { + // fmt.Println(string(annPatches)) + ws.annotationsController.Add(rkind, rns, rname, annPatches) + } ok, msg := isAdmSuccesful(policyInfos) if ok { - patches := engine.JoinPatches(allPatches) - if len(annPatches) > 0 { - patches, err = jsonpatch.MergePatch(patches, annPatches) - if err != nil { - glog.Error(err) - } - } patchType := v1beta1.PatchTypeJSONPatch return &v1beta1.AdmissionResponse{ Allowed: true, - Patch: patches, + Patch: engine.JoinPatches(allPatches), PatchType: &patchType, } } diff --git a/pkg/webhooks/report.go b/pkg/webhooks/report.go index a80667db41..7f9b4697af 100644 --- a/pkg/webhooks/report.go +++ b/pkg/webhooks/report.go @@ -1,7 +1,6 @@ package webhooks import ( - "fmt" "strings" "github.com/nirmata/kyverno/pkg/annotations" @@ -69,7 +68,6 @@ func addAnnotationsToResource(rawResource []byte, pi *info.PolicyInfo, ruleType } // get annotations ann := annotations.ParseAnnotationsFromObject(rawResource) - fmt.Println(ann) ann, patch, err := annotations.AddPolicyJSONPatch(ann, pi, ruleType) if err != nil { glog.Error(err) diff --git a/pkg/webhooks/utils.go b/pkg/webhooks/utils.go index b29cd16449..238222216d 100644 --- a/pkg/webhooks/utils.go +++ b/pkg/webhooks/utils.go @@ -131,6 +131,5 @@ func checkIfOnlyAnnotationsUpdate(request *v1beta1.AdmissionRequest) bool { if reflect.DeepEqual(objUnstr, oldobjUnstr) { return true } - return false } diff --git a/test/StatefulSet/policy-StatefulSet.yaml b/test/StatefulSet/policy-StatefulSet.yaml index 2460a1235c..00b31b3225 100644 --- a/test/StatefulSet/policy-StatefulSet.yaml +++ b/test/StatefulSet/policy-StatefulSet.yaml @@ -28,7 +28,7 @@ spec: message: "This SS is broken" pattern: spec: - replicas: ">20" + replicas: ">2" volumeClaimTemplates: - metadata: name: www From 15918ec0d802d2223e9144eefaf28ef5576394bb Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Fri, 19 Jul 2019 20:39:31 -0700 Subject: [PATCH 23/24] rebase with master --- definitions/install.yaml | 2 +- examples/demo/1_image_pull_policy/nginx.yaml | 21 ++++++ examples/demo/1_image_pull_policy/policy.yaml | 26 +++++++ examples/demo/2_allowed_registry/nginx.yaml | 21 ++++++ examples/demo/2_allowed_registry/policy.yaml | 22 ++++++ .../namespace.yaml | 0 .../policy.yaml | 9 ++- examples/demo/4_non_root/nginx.yaml | 21 ++++++ examples/demo/4_non_root/policy.yaml | 21 ++++++ examples/demo/5_health_check/pod.yaml | 32 +++++++++ examples/demo/5_health_check/policy.yaml | 33 +++++++++ examples/demo/{qos => 6_qos}/policy_qos.yaml | 0 examples/demo/{qos => 6_qos}/qos.yaml | 2 +- .../7_container_security_context/nginx.yaml | 22 ++++++ .../7_container_security_context/policy.yaml | 26 +++++++ examples/demo/mutate_patch/ep.yaml | 13 ---- examples/demo/mutate_patch/policy_patch.yaml | 24 ------- pkg/engine/overlay_new.go | 70 +++++++++++++++++++ .../SecurityContext}/nginx.yaml | 0 .../SecurityContext}/policy.yaml | 0 20 files changed, 324 insertions(+), 41 deletions(-) create mode 100644 examples/demo/1_image_pull_policy/nginx.yaml create mode 100644 examples/demo/1_image_pull_policy/policy.yaml create mode 100644 examples/demo/2_allowed_registry/nginx.yaml create mode 100644 examples/demo/2_allowed_registry/policy.yaml rename examples/demo/{generate => 3_network_policy}/namespace.yaml (100%) rename examples/demo/{generate => 3_network_policy}/policy.yaml (70%) create mode 100644 examples/demo/4_non_root/nginx.yaml create mode 100644 examples/demo/4_non_root/policy.yaml create mode 100644 examples/demo/5_health_check/pod.yaml create mode 100644 examples/demo/5_health_check/policy.yaml rename examples/demo/{qos => 6_qos}/policy_qos.yaml (100%) rename examples/demo/{qos => 6_qos}/qos.yaml (92%) create mode 100755 examples/demo/7_container_security_context/nginx.yaml create mode 100755 examples/demo/7_container_security_context/policy.yaml delete mode 100644 examples/demo/mutate_patch/ep.yaml delete mode 100644 examples/demo/mutate_patch/policy_patch.yaml create mode 100755 pkg/engine/overlay_new.go rename {examples/demo/security_context => test/SecurityContext}/nginx.yaml (100%) rename {examples/demo/security_context => test/SecurityContext}/policy.yaml (100%) diff --git a/definitions/install.yaml b/definitions/install.yaml index 7b44e59dc8..f076a730c7 100644 --- a/definitions/install.yaml +++ b/definitions/install.yaml @@ -184,7 +184,7 @@ spec: containers: - name: kyverno image: nirmata/kyverno:latest - args: ["--filterKind","Nodes,Events,APIService,SubjectAccessReview"] + args: ["--filterKind","Node,Event,APIService,Policy,TokenReview,SubjectAccessReview"] ports: - containerPort: 443 securityContext: diff --git a/examples/demo/1_image_pull_policy/nginx.yaml b/examples/demo/1_image_pull_policy/nginx.yaml new file mode 100644 index 0000000000..c3bdbed5d6 --- /dev/null +++ b/examples/demo/1_image_pull_policy/nginx.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx + cli: test +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:latest + # imagePullPolicy: IfNotPresent diff --git a/examples/demo/1_image_pull_policy/policy.yaml b/examples/demo/1_image_pull_policy/policy.yaml new file mode 100644 index 0000000000..6da37a961e --- /dev/null +++ b/examples/demo/1_image_pull_policy/policy.yaml @@ -0,0 +1,26 @@ +apiVersion : kyverno.io/v1alpha1 +kind: Policy +metadata: + name: image-pull-policy +spec: + rules: + - name: image-pull-policy + resource: + kinds: + - Deployment + # - StatefulSet + # name: "my-deployment" + # selector : + # matchLabels: + # app.type: prod + # namespace: "my-namespace" + mutate: + overlay: + spec: + template: + spec: + containers: + # select images which end with :latest + - (image): "*latest" + # require that the imagePullPolicy is "IfNotPresent" + imagePullPolicy: IfNotPresent diff --git a/examples/demo/2_allowed_registry/nginx.yaml b/examples/demo/2_allowed_registry/nginx.yaml new file mode 100644 index 0000000000..a0329d80f4 --- /dev/null +++ b/examples/demo/2_allowed_registry/nginx.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx + cli: test +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + # image: nginx + image: nirmata/nginx diff --git a/examples/demo/2_allowed_registry/policy.yaml b/examples/demo/2_allowed_registry/policy.yaml new file mode 100644 index 0000000000..4964e434d9 --- /dev/null +++ b/examples/demo/2_allowed_registry/policy.yaml @@ -0,0 +1,22 @@ +apiVersion : kyverno.io/v1alpha1 +kind: Policy +metadata: + name: check-registries +spec: + rules: + - name: check-registries + resource: + kinds: + - Deployment + - StatefulSet + namespace: default + validate: + message: "Registry is not allowed" + pattern: + spec: + template: + spec: + containers: + - name: "*" + # Check allowed registries + image: "*nirmata* | https://private.registry.io/*" diff --git a/examples/demo/generate/namespace.yaml b/examples/demo/3_network_policy/namespace.yaml similarity index 100% rename from examples/demo/generate/namespace.yaml rename to examples/demo/3_network_policy/namespace.yaml diff --git a/examples/demo/generate/policy.yaml b/examples/demo/3_network_policy/policy.yaml similarity index 70% rename from examples/demo/generate/policy.yaml rename to examples/demo/3_network_policy/policy.yaml index ed2465c7fd..c68090abf6 100644 --- a/examples/demo/generate/policy.yaml +++ b/examples/demo/3_network_policy/policy.yaml @@ -8,7 +8,7 @@ spec: resource: kinds: - Namespace - name: "*" + name: "devtest" generate: kind: NetworkPolicy name: deny-ingress-traffic @@ -22,4 +22,9 @@ spec: metadata: annotations: {} labels: - policyname: "default" \ No newline at end of file + policyname: "default" + # kind: ConfigMap + # name: default-config + # clone: + # namespace: default + # name: config-template \ No newline at end of file diff --git a/examples/demo/4_non_root/nginx.yaml b/examples/demo/4_non_root/nginx.yaml new file mode 100644 index 0000000000..41c00d3066 --- /dev/null +++ b/examples/demo/4_non_root/nginx.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: psp-demo-unprivileged + labels: + app.type: prod +spec: + replicas: 1 + selector: + matchLabels: + app: psp + template: + metadata: + labels: + app: psp + spec: + securityContext: + runAsNonRoot: true + containers: + - name: sec-ctx-unprivileged + image: nginxinc/nginx-unprivileged diff --git a/examples/demo/4_non_root/policy.yaml b/examples/demo/4_non_root/policy.yaml new file mode 100644 index 0000000000..3ea11b319e --- /dev/null +++ b/examples/demo/4_non_root/policy.yaml @@ -0,0 +1,21 @@ +apiVersion : kyverno.io/v1alpha1 +kind: Policy +metadata: + name: policy-security-context +spec: + rules: + - name: validate-runAsNonRoot + resource: + kinds: + - Deployment + selector : + matchLabels: + app.type: prod + validate: + message: "security context 'runAsNonRoot' shoud be set to true" + pattern: + spec: + template: + spec: + securityContext: + runAsNonRoot: true \ No newline at end of file diff --git a/examples/demo/5_health_check/pod.yaml b/examples/demo/5_health_check/pod.yaml new file mode 100644 index 0000000000..f5f0004d3a --- /dev/null +++ b/examples/demo/5_health_check/pod.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + test: probe + name: probe +spec: + containers: + - name: readiness + image: k8s.gcr.io/busybox + args: + - /bin/sh + - -c + - touch /tmp/healthy; sleep 30; rm -rf /tmp/healthy; sleep 600 + readinessProbe: + # successThreshold: 3 + exec: + command: + - cat + - /tmp/healthy + - name: liveness + image: k8s.gcr.io/liveness + args: + - /server + livenessProbe: + httpGet: + path: /healthz + port: 8080 + httpHeaders: + - name: Custom-Header + value: Awesome + periodSeconds: 3 \ No newline at end of file diff --git a/examples/demo/5_health_check/policy.yaml b/examples/demo/5_health_check/policy.yaml new file mode 100644 index 0000000000..0550b16fe6 --- /dev/null +++ b/examples/demo/5_health_check/policy.yaml @@ -0,0 +1,33 @@ +apiVersion : kyverno.io/v1alpha1 +kind : Policy +metadata : + name: check-probe-exists +spec: + rules: + - name: check-readinessProbe-exists + resource: + kinds : + - Pod + validate: + message: "readinessProbe is required" + pattern: + spec: + containers: + - (name): "readiness" + readinessProbe: + successThreshold: ">1" + - name: check-livenessProbe-exists + resource: + kinds : + - Pod + validate: + message: "livenessProbe is required" + pattern: + spec: + containers: + - (name): "liveness" + livenessProbe: + httpGet: + path: "?*" + port: "*" + scheme: "?*" diff --git a/examples/demo/qos/policy_qos.yaml b/examples/demo/6_qos/policy_qos.yaml similarity index 100% rename from examples/demo/qos/policy_qos.yaml rename to examples/demo/6_qos/policy_qos.yaml diff --git a/examples/demo/qos/qos.yaml b/examples/demo/6_qos/qos.yaml similarity index 92% rename from examples/demo/qos/qos.yaml rename to examples/demo/6_qos/qos.yaml index c81b65b0e5..d998bdfbc3 100644 --- a/examples/demo/qos/qos.yaml +++ b/examples/demo/6_qos/qos.yaml @@ -17,4 +17,4 @@ spec: image: nginx:latest resources: limits: - cpu: "50m" \ No newline at end of file + cpu: "50m" diff --git a/examples/demo/7_container_security_context/nginx.yaml b/examples/demo/7_container_security_context/nginx.yaml new file mode 100755 index 0000000000..811f167bac --- /dev/null +++ b/examples/demo/7_container_security_context/nginx.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: psp-demo-unprivileged + labels: + app.type: prod +spec: + replicas: 1 + selector: + matchLabels: + app: psp + template: + metadata: + labels: + app: psp + spec: + containers: + - name: sec-ctx-unprivileged + image: nginxinc/nginx-unprivileged + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: true diff --git a/examples/demo/7_container_security_context/policy.yaml b/examples/demo/7_container_security_context/policy.yaml new file mode 100755 index 0000000000..fc434ec0dc --- /dev/null +++ b/examples/demo/7_container_security_context/policy.yaml @@ -0,0 +1,26 @@ +apiVersion : kyverno.io/v1alpha1 +kind: Policy +metadata: + name: container-security-context +spec: + rules: + - name: validate-user-privilege + resource: + kinds: + - Deployment + selector : + matchLabels: + app.type: prod + validate: + message: "validate container security contexts" + pattern: + spec: + template: + spec: + containers: + - securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + # fields can be customized + # privileged: false + # readOnlyRootFilesystem: true \ No newline at end of file diff --git a/examples/demo/mutate_patch/ep.yaml b/examples/demo/mutate_patch/ep.yaml deleted file mode 100644 index f932126c22..0000000000 --- a/examples/demo/mutate_patch/ep.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Endpoints -metadata: - name: demo-endpoint - labels: - label : test -subsets: -- addresses: - - ip: 192.168.10.171 - ports: - - name: secure-connection - port: 443 - protocol: TCP diff --git a/examples/demo/mutate_patch/policy_patch.yaml b/examples/demo/mutate_patch/policy_patch.yaml deleted file mode 100644 index 6ef8e9f0c7..0000000000 --- a/examples/demo/mutate_patch/policy_patch.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion : kyverno.io/v1alpha1 -kind : Policy -metadata : - name : policy-endpoints -spec : - rules: - - name: demo-ep - resource: - kinds : - - Endpoints - selector: - matchLabels: - label : test - name: demo-endpoint - mutate: - patches: - # add a new label - - path: /metadata/labels/app.type - op: add - value: dev - # replace port - - path : /subsets/0/ports/0/port - op : replace - value: 9663 \ No newline at end of file diff --git a/pkg/engine/overlay_new.go b/pkg/engine/overlay_new.go new file mode 100755 index 0000000000..8f807212f1 --- /dev/null +++ b/pkg/engine/overlay_new.go @@ -0,0 +1,70 @@ +package engine + +import ( + "reflect" +) + +// func processoverlay(rule kubepolicy.Rule, rawResource []byte, gvk metav1.GroupVersionKind) ([][]byte, error) { + +// var resource interface{} +// var appliedPatches [][]byte +// err := json.Unmarshal(rawResource, &resource) +// if err != nil { +// return nil, err +// } + +// patches, err := mutateResourceWithOverlay(resource, *rule.Mutation.Overlay) +// if err != nil { +// return nil, err +// } +// appliedPatches = append(appliedPatches, patches...) + +// return appliedPatches, err +// } + +func applyoverlay(resource, overlay interface{}, path string) ([][]byte, error) { + var appliedPatches [][]byte + // resource item exists but has different type - replace + // all subtree within this path by overlay + if reflect.TypeOf(resource) != reflect.TypeOf(overlay) { + patch, err := replaceSubtree(overlay, path) + if err != nil { + return nil, err + } + + appliedPatches = append(appliedPatches, patch) + } + + return applyOverlayForSameTypes(resource, overlay, path) +} + +func checkConditions(resource, overlay interface{}, path string) bool { + + switch typedOverlay := overlay.(type) { + case map[string]interface{}: + typedResource := resource.(map[string]interface{}) + if !checkConditionOnMap(typedResource, typedOverlay) { + return false + } + case []interface{}: + typedResource := resource.([]interface{}) + if !checkConditionOnArray(typedResource, typedOverlay) { + return false + } + case string, float64, int64, bool: + + default: + return false + } + return true +} + +func checkConditionOnMap(resourceMap, overlayMap map[string]interface{}) bool { + // _ := getAnchorsFromMap(overlayMap) + + return false +} + +func checkConditionOnArray(resource, overlay []interface{}) bool { + return false +} diff --git a/examples/demo/security_context/nginx.yaml b/test/SecurityContext/nginx.yaml similarity index 100% rename from examples/demo/security_context/nginx.yaml rename to test/SecurityContext/nginx.yaml diff --git a/examples/demo/security_context/policy.yaml b/test/SecurityContext/policy.yaml similarity index 100% rename from examples/demo/security_context/policy.yaml rename to test/SecurityContext/policy.yaml From bbed4510392431539797dbb36f51601f9b7aaabe Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Sat, 20 Jul 2019 01:11:25 -0700 Subject: [PATCH 24/24] cleanup --- pkg/annotations/annotations.go | 1 - pkg/dclient/utils.go | 215 --------------------------------- pkg/engine/engine.go | 1 - pkg/engine/mutation.go | 4 - 4 files changed, 221 deletions(-) diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index e2dd9d2df3..34bbfe1177 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -88,7 +88,6 @@ func (p *Policy) updatePolicy(obj *Policy, ruleType pinfo.RuleType) bool { updates = true } } - p.Status = obj.Status // check if any rules failed p.Status = p.getOverAllStatus() // If there are any updates then the annotation can be updated, can skip diff --git a/pkg/dclient/utils.go b/pkg/dclient/utils.go index c982c24574..6492187038 100644 --- a/pkg/dclient/utils.go +++ b/pkg/dclient/utils.go @@ -1,12 +1,9 @@ package client import ( - "errors" "strings" "time" - "github.com/golang/glog" - "github.com/nirmata/kyverno/pkg/info" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -113,215 +110,3 @@ func retry(attempts int, sleep time.Duration, fn func() error) error { type stop struct { error } - -func GetAnnotations(obj *unstructured.Unstructured) map[string]interface{} { - var annotationsMaps map[string]interface{} - unstr := obj.UnstructuredContent() - metadata, ok := unstr["metadata"] - if ok { - metadataMap, ok := metadata.(map[string]interface{}) - if !ok { - glog.Info("type mismatch") - return nil - } - annotations, ok := metadataMap["annotations"] - if !ok { - glog.Info("annotations not present") - return nil - } - annotationsMaps, ok = annotations.(map[string]interface{}) - if !ok { - glog.Info("type mismatch") - return nil - } - } - return annotationsMaps -} - -func SetAnnotations(obj *unstructured.Unstructured, annotations map[string]interface{}) error { - unstr := obj.UnstructuredContent() - metadata, ok := unstr["metadata"] - if ok { - metadataMap, ok := metadata.(map[string]interface{}) - if !ok { - return errors.New("type mismatch") - } - metadataMap["annotations"] = annotations - unstr["metadata"] = metadataMap - obj.SetUnstructuredContent(unstr) - } - return nil -} - -type AnnotationPolicies struct { - // map[policy_name] - Policies map[string]AnnotationPolicy `json:"policies"` -} - -type AnnotationPolicy struct { - Status string `json:"status"` - Rules []AnnotationRule `json:"rules,omitempty"` -} - -type AnnotationRule struct { - Name string `json:"name"` - Status string `json:"status"` - Type string `json:"type"` - Changes string `json:"changes"` -} - -func getStatus(status bool) string { - if status { - return "Success" - } - return "Failure" -} - -func getRules(rules []*info.RuleInfo) []AnnotationRule { - var annrules []AnnotationRule - for _, r := range rules { - annrule := AnnotationRule{Name: r.Name, - Status: getStatus(r.IsSuccessful())} - //TODO: add mutation changes in policyInfo and in annotation - annrules = append(annrules, annrule) - } - return annrules -} - -// input rules can be mutation or validation -func (ap AnnotationPolicy) updateRules(rules interface{}, validation bool) (error, interface{}) { - ruleList, ok := rules.([]interface{}) - updated := false - if !ok { - return errors.New("type mismatch"), false - } - - // for mutation rule check if the rules are same - // var mode string - // if validation { - // mode = "Validation" - // } else { - // mode = "Mutation" - // } - // // if lengths are differrent then update - // if len(ruleList) != len(ap.Rules) { - // return nil, ap.updateRules - // } - // check if there is any update in the rules - // order of rules is assumed same while comparison - for i, r := range ruleList { - rule, ok := r.(map[string]interface{}) - if !ok { - return errors.New("type mismatch"), nil - } - // Name - name, ok := rule["name"].(string) - if !ok { - return errors.New("type mismatch"), nil - } - if name != ap.Rules[i].Name { - updated = true - break - } - // Status - status, ok := rule["status"].(string) - if !ok { - return errors.New("type mismatch"), nil - } - if status != ap.Rules[i].Status { - updated = true - break - } - } - if updated { - return nil, ap.Rules - } - return nil, nil -} - -func newAnnotationPolicy(pi *info.PolicyInfo) AnnotationPolicy { - status := getStatus(pi.IsSuccessful()) - rules := getRules(pi.Rules) - return AnnotationPolicy{Status: status, - Rules: rules} -} - -//func GetPolicies(policies interface{}) map[string] -func AddPolicy(pi *info.PolicyInfo, ann map[string]interface{}, validation bool) (error, map[string]interface{}) { - // Lets build the policy annotation struct from policyInfo - annpolicy := newAnnotationPolicy(pi) - // Add policy to annotations - // If policy does not exist -> Add - // If already exists then update the status and rules - policies, ok := ann["policies"] - if ok { - policiesMap, ok := policies.(map[string]interface{}) - if !ok { - glog.Info("type mismatch") - return errors.New("type mismatch"), nil - } - // check if policy record is present - policy, ok := policiesMap[pi.Name] - if !ok { - // not present then we add - policiesMap[pi.Name] = annpolicy - ann["policies"] = policiesMap - return nil, ann - } - policyMap, ok := policy.(map[string]interface{}) - if !ok { - return errors.New("type mismatch"), nil - } - // We just update the annotations - // status - status := policyMap["status"] - statusStr, ok := status.(string) - if !ok { - return errors.New("type mismatch"), nil - } - if statusStr != annpolicy.Status { - policyMap["status"] = annpolicy.Status - } - // check rules - rules, ok := policyMap["rules"] - if !ok { - return errors.New("no rules"), nil - } - err, newRules := annpolicy.updateRules(rules, validation) - if err != nil { - return err, nil - } - if newRules == nil { - //nothing to update - return nil, nil - } - // update the new rule - policyMap["rules"] = newRules - // update policies map - policiesMap[pi.Name] = policyMap - ann["policies"] = policiesMap - return nil, ann - } - return nil, nil -} - -// RemovePolicy -func RemovePolicy(pi *info.PolicyInfo, ann map[string]interface{}) (error, map[string]interface{}) { - policies, ok := ann["policies"] - if ok { - policiesMap, ok := policies.(map[string]interface{}) - if !ok { - glog.Info("type mismatch") - return errors.New("type mismatch"), nil - } - // check if policy record is present - _, ok = policiesMap[pi.Name] - if ok { - // delete the pair - delete(policiesMap, pi.Name) - ann["policies"] = policiesMap - return nil, ann - } - } - return nil, nil -} diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index ed938048bc..5dfed4c162 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -52,7 +52,6 @@ func ProcessExisting(client *client.Client, policy *types.Policy) []*info.Policy ri := &resourceInfo{resource: &res, gvk: &metav1.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind}} - // resources = append(resources, ri) resourceMap[string(res.GetUID())] = ri diff --git a/pkg/engine/mutation.go b/pkg/engine/mutation.go index a1f490d482..2721a4746e 100644 --- a/pkg/engine/mutation.go +++ b/pkg/engine/mutation.go @@ -32,8 +32,6 @@ func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersio ri.Addf("overlay application has failed, err %v.", err) } else { ri.Addf("Rule %s: Overlay succesfully applied.", rule.Name) - //TODO: patchbytes -> string - //glog.V(3).Info(" Overlay succesfully applied. Patch %s", string(overlayPatches)) allPatches = append(allPatches, overlayPatches...) } } @@ -48,8 +46,6 @@ func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersio } } else { ri.Addf("Rule %s: Patches succesfully applied.", rule.Name) - //TODO: patchbytes -> string - //glog.V(3).Info("Patches succesfully applied. Patch %s", string(overlayPatches)) allPatches = append(allPatches, rulePatches...) } }