annotations creation,update & removal

pkg/annotations/annotations.go

@@ -22,7 +22,7 @@ type Policy struct {
 //Rule information for annotations
 type Rule struct {
 	Status  string `json:"status"`
-	Changes string `json:"changes"`
+	Changes string `json:"changes,omitempty"` // TODO for mutation changes
 }
 
 func getStatus(status bool) string {
@@ -187,6 +187,9 @@ func AddPolicy(obj *unstructured.Unstructured, pi *pinfo.PolicyInfo, ruleType pi
 func RemovePolicy(obj *unstructured.Unstructured, policy string) bool {
 	// get annotations
 	ann := obj.GetAnnotations()
+	if ann == nil {
+		return false
+	}
 	if _, ok := ann[policy]; !ok {
 		return false
 	}
@@ -205,10 +208,11 @@ func ParseAnnotationsFromObject(bytes []byte) map[string]string {
 		glog.Error("unable to parse")
 		return nil
 	}
-	if annotations, ok := meta["annotations"].(map[string]string); ok {
-		return annotations
+	ann, ok, err := unstructured.NestedStringMap(meta, "annotations")
+	if err != nil || !ok {
+		return nil
 	}
-	return nil
+	return ann
 }
 
 //AddPolicyJSONPatch generate JSON Patch to add policy informatino JSON patch
@@ -227,6 +231,12 @@ func AddPolicyJSONPatch(ann map[string]string, pi *pinfo.PolicyInfo, ruleType pi
 		ann[pi.Name] = string(PolicyByte)
 		// create add JSON patch
 		jsonPatch, err := createAddJSONPatch(ann)
+		// var jsonPatch []byte
+		// if len(ann) == 0 {
+		// 	jsonPatch, err = createAddJSONPatch(ann)
+		// } else {
+		// 	jsonPatch, err = createReplaceJSONPatch(ann)
+		// }
 		return ann, jsonPatch, err
 	}
 	cPolicyObj := Policy{}
@@ -234,8 +244,8 @@ func AddPolicyJSONPatch(ann map[string]string, pi *pinfo.PolicyInfo, ruleType pi
 	// update policy information inside the annotation
 	// 1> policy status
 	// 2> rule (name, status,changes,type)
-	cPolicyObj.updatePolicy(PolicyObj, ruleType)
-	if err != nil {
+	update := cPolicyObj.updatePolicy(PolicyObj, ruleType)
+	if !update {
 		return nil, nil, err
 	}
 	cPolicyByte, err := json.Marshal(cPolicyObj)

pkg/annotations/controller.go

@@ -34,7 +34,7 @@ func NewAnnotationControler(client *client.Client) Controller {
 }
 
 func (c *controller) Add(rkind, rns, rname string, patch []byte) {
-	c.queue.Add(newInfo)
+	c.queue.Add(newInfo(rkind, rns, rname, patch))
 }
 
 func (c *controller) Run(stopCh <-chan struct{}) {
@@ -55,35 +55,49 @@ func (c *controller) runWorker() {
 	}
 }
 
-func (c *controller) processNextWorkItem() bool {
-	obj, shutdown := c.queue.Get()
+func (pc *controller) processNextWorkItem() bool {
+	obj, shutdown := pc.queue.Get()
 	if shutdown {
 		return false
 	}
+
 	err := func(obj interface{}) error {
-		defer c.queue.Done(obj)
-		var key info
-		var ok bool
-		if key, ok = obj.(info); !ok {
-			c.queue.Forget(obj)
-			glog.Warningf("Expecting type info by got %v\n", obj)
-			return nil
-		}
-		// Run the syncHandler, passing the resource and the policy
-		if err := c.SyncHandler(key); err != nil {
-			c.queue.AddRateLimited(key)
-			return fmt.Errorf("error syncing '%s/%s/%s' : %s, requeuing annotation creation request", key.RKind, key.RNs, key.RName, err)
-		}
+		defer pc.queue.Done(obj)
+		err := pc.syncHandler(obj)
+		pc.handleErr(err, obj)
 		return nil
 	}(obj)
-
 	if err != nil {
-		glog.Warning(err)
+		glog.Error(err)
+		return true
 	}
 	return true
 }
+func (pc *controller) handleErr(err error, key interface{}) {
+	if err == nil {
+		pc.queue.Forget(key)
+		return
+	}
+	// This controller retries if something goes wrong. After that, it stops trying.
+	if pc.queue.NumRequeues(key) < WorkQueueRetryLimit {
+		glog.Warningf("Error syncing events %v: %v", key, err)
+		// Re-enqueue the key rate limited. Based on the rate limiter on the
+		// queue and the re-enqueue history, the key will be processed later again.
+		pc.queue.AddRateLimited(key)
+		return
+	}
+	pc.queue.Forget(key)
+	glog.Error(err)
+	glog.Warningf("Dropping the key %q out of the queue: %v", key, err)
+}
+
+func (c *controller) syncHandler(obj interface{}) error {
+	var key info
+	var ok bool
+	if key, ok = obj.(info); !ok {
+		return fmt.Errorf("expected string in workqueue but got %#v", obj)
+	}
 
-func (c *controller) SyncHandler(key info) error {
 	var err error
 	// check if the resource is created
 	_, err = c.client.GetResource(key.RKind, key.RNs, key.RName)
@@ -92,7 +106,7 @@ func (c *controller) SyncHandler(key info) error {
 		return err
 	}
 	// if it is patch the resource
-	_, err = c.client.PatchResource(key.RKind, key.RNs, key.RName, key.Patch)
+	_, err = c.client.PatchResource(key.RKind, key.RNs, key.RName, *key.Patch)
 	if err != nil {
 		glog.Errorf("Error creating annotation: unable to get resource %s/%s/%s, will retry: %s", key.RKind, key.RNs, key.RName, err)
 		return err

pkg/annotations/utils.go

@@ -2,19 +2,21 @@ package annotations
 
 const annotationQueueName = "annotation-queue"
 const workerThreadCount = 1
+const WorkQueueRetryLimit = 3
 
 type info struct {
 	RKind string
 	RNs   string
 	RName string
-	Patch []byte
+	//TODO:Hack as slice makes the struct unhasable
+	Patch *[]byte
 }
 
 func newInfo(rkind, rns, rname string, patch []byte) info {
 	return info{
 		RKind: rkind,
-		RNs:   rname,
+		RNs:   rns,
 		RName: rname,
-		Patch: patch,
+		Patch: &patch,
 	}
 }

pkg/controller/cleanup.go

@@ -0,0 +1,78 @@
+package controller
+
+import (
+	"github.com/golang/glog"
+	"github.com/minio/minio/pkg/wildcard"
+	"github.com/nirmata/kyverno/pkg/annotations"
+	v1alpha1 "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1"
+	client "github.com/nirmata/kyverno/pkg/dclient"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+func cleanAnnotations(client *client.Client, obj interface{}) {
+	// get the policy struct from interface
+	unstr, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
+	if err != nil {
+		glog.Error(err)
+		return
+	}
+	policy := v1alpha1.Policy{}
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstr, &policy); err != nil {
+		glog.Error(err)
+		return
+	}
+	// Get the resources that apply to the policy
+	// key uid
+	resourceMap := map[string]unstructured.Unstructured{}
+	for _, rule := range policy.Spec.Rules {
+		for _, k := range rule.Kinds {
+			if k == "Namespace" {
+				// REWORK: will be handeled by namespace controller
+				continue
+			}
+			// kind -> resource
+			gvr := client.DiscoveryClient.GetGVRFromKind(k)
+			// label selectors
+			// namespace ? should it be default or allow policy to specify it
+			namespace := "default"
+			if rule.ResourceDescription.Namespace != nil {
+				namespace = *rule.ResourceDescription.Namespace
+			}
+			list, err := client.ListResource(k, namespace, rule.ResourceDescription.Selector)
+			if err != nil {
+				glog.Errorf("unable to list resource for %s with label selector %s", gvr.Resource, rule.Selector.String())
+				glog.Errorf("unable to apply policy %s rule %s. err: %s", policy.Name, rule.Name, err)
+				continue
+			}
+			for _, res := range list.Items {
+				name := rule.ResourceDescription.Name
+				if name != nil {
+					// wild card matching
+					if !wildcard.Match(*name, res.GetName()) {
+						continue
+					}
+				}
+				resourceMap[string(res.GetUID())] = res
+			}
+		}
+	}
+
+	// remove annotations for the resources
+	for _, obj := range resourceMap {
+		// get annotations
+		ann := obj.GetAnnotations()
+
+		_, patch, err := annotations.RemovePolicyJSONPatch(ann, policy.Name)
+		if err != nil {
+			glog.Error(err)
+			continue
+		}
+		// patch the resource
+		_, err = client.PatchResource(obj.GetKind(), obj.GetNamespace(), obj.GetName(), patch)
+		if err != nil {
+			glog.Error(err)
+			continue
+		}
+	}
+}

pkg/controller/controller.go

@@ -88,6 +88,7 @@ func (pc *PolicyController) deletePolicyHandler(resource interface{}) {
 		return
 	}
 	//TODO: need to clear annotations on the resources
+	cleanAnnotations(pc.client, resource)
 	glog.Infof("policy deleted: %s", object.GetName())
 }
 
@@ -191,12 +192,12 @@ func (pc *PolicyController) syncHandler(obj interface{}) error {
 	//TODO: processPolicy
 	glog.Infof("process policy %s on existing resources", policy.GetName())
 	policyInfos := engine.ProcessExisting(pc.client, policy)
-	events, violations := pc.createEventsAndViolations(policyInfos)
+	events, _ := pc.createEventsAndViolations(policyInfos)
 	pc.eventController.Add(events...)
-	err = pc.violationBuilder.Add(violations...)
-	if err != nil {
-		glog.Error(err)
-	}
+	// err = pc.violationBuilder.Add(violations...)
+	// if err != nil {
+	// 	glog.Error(err)
+	// }
 
 	// add annotations to resources
 	pc.createAnnotations(policyInfos)
@@ -241,10 +242,9 @@ func (pc *PolicyController) createAnnotations(policyInfos []*info.PolicyInfo) {
 		if mpatch == nil {
 			patch = vpatch
 		} else {
-			patch = vpatch
+			patch = mpatch
 		}
-
-		// add the anotation to the resource
+		//		add the anotation to the resource
 		_, err = pc.client.PatchResource(pi.RKind, pi.RNamespace, pi.RName, patch)
 		if err != nil {
 			glog.Error(err)

pkg/engine/engine.go

@@ -14,11 +14,11 @@ import (
 // Instead we expose them as standalone functions passing the required atrributes
 // The each function returns the changes that need to be applied on the resource
 // the caller is responsible to apply the changes to the resource
-
 // ProcessExisting checks for mutation and validation violations of existing resources
 func ProcessExisting(client *client.Client, policy *types.Policy) []*info.PolicyInfo {
 	glog.Infof("Applying policy %s on existing resources", policy.Name)
-	resources := []*resourceInfo{}
+	// key uid
+	resourceMap := map[string]*resourceInfo{}
 
 	for _, rule := range policy.Spec.Rules {
 		for _, k := range rule.Kinds {
@@ -52,18 +52,20 @@ func ProcessExisting(client *client.Client, policy *types.Policy) []*info.Policy
 				ri := &resourceInfo{resource: &res, gvk: &metav1.GroupVersionKind{Group: gvk.Group,
 					Version: gvk.Version,
 					Kind:    gvk.Kind}}
-				resources = append(resources, ri)
+				// resources = append(resources, ri)
+
+				resourceMap[string(res.GetUID())] = ri
 
 			}
 		}
 	}
 	policyInfos := []*info.PolicyInfo{}
 	// for the filtered resource apply policy
-	for _, r := range resources {
+	for _, v := range resourceMap {
 
-		policyInfo, err := applyPolicy(client, policy, r)
+		policyInfo, err := applyPolicy(client, policy, v)
 		if err != nil {
-			glog.Errorf("unable to apply policy %s on resource %s/%s", policy.Name, r.resource.GetName(), r.resource.GetNamespace())
+			glog.Errorf("unable to apply policy %s on resource %s/%s", policy.Name, v.resource.GetName(), v.resource.GetNamespace())
 			glog.Error(err)
 			continue
 		}

pkg/webhooks/deleteresource.go

@@ -9,7 +9,7 @@ import (
 )
 
 func (ws *WebhookServer) removePolicyViolation(request *v1beta1.AdmissionRequest) error {
-
+	return nil
 	// Get the list of policies that apply on the resource
 	policies, err := ws.policyLister.List(labels.NewSelector())
 	if err != nil {

pkg/webhooks/mutation.go

@@ -1,8 +1,6 @@
 package webhooks
 
 import (
-	"fmt"
-
 	jsonpatch "github.com/evanphx/json-patch"
 
 	"github.com/golang/glog"
@@ -59,10 +57,10 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be
 			}
 		} else {
 			// CleanUp Violations if exists
-			err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Mutation)
-			if err != nil {
-				glog.Info(err)
-			}
+			// err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Mutation)
+			// if err != nil {
+			// 	glog.Info(err)
+			// }
 
 			if len(policyPatches) > 0 {
 				allPatches = append(allPatches, policyPatches...)
@@ -78,8 +76,6 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be
 				annPatches, err = jsonpatch.MergePatch(annPatches, annPatch)
 				if err != nil {
 					glog.Error(err)
-					fmt.Println("Mergining docs")
-					fmt.Println(err)
 				}
 			}
 		}
@@ -96,7 +92,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be
 		if len(annPatches) > 0 {
 			patches, err = jsonpatch.MergePatch(patches, annPatches)
 			if err != nil {
-				fmt.Println(err)
+				glog.Error(err)
 			}
 		}
 		patchType := v1beta1.PatchTypeJSONPatch

pkg/webhooks/report.go

@@ -1,6 +1,7 @@
 package webhooks
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/nirmata/kyverno/pkg/annotations"
@@ -63,8 +64,12 @@ func newEventInfoFromPolicyInfo(policyInfoList []*info.PolicyInfo, onUpdate bool
 }
 
 func addAnnotationsToResource(rawResource []byte, pi *info.PolicyInfo, ruleType info.RuleType) []byte {
+	if len(pi.Rules) == 0 {
+		return nil
+	}
 	// get annotations
 	ann := annotations.ParseAnnotationsFromObject(rawResource)
+	fmt.Println(ann)
 	ann, patch, err := annotations.AddPolicyJSONPatch(ann, pi, ruleType)
 	if err != nil {
 		glog.Error(err)

pkg/webhooks/validation.go

@@ -6,7 +6,6 @@ import (
 	engine "github.com/nirmata/kyverno/pkg/engine"
 	"github.com/nirmata/kyverno/pkg/event"
 	"github.com/nirmata/kyverno/pkg/info"
-	"github.com/nirmata/kyverno/pkg/violation"
 	v1beta1 "k8s.io/api/admission/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -16,7 +15,7 @@ import (
 // If there are no errors in validating rule we apply generation rules
 func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse {
 	policyInfos := []*info.PolicyInfo{}
-	var violations []*violation.Info
+	// var violations []*violation.Info
 	var eventsInfo []*event.Info
 	policies, err := ws.policyLister.List(labels.NewSelector())
 	if err != nil {
@@ -27,6 +26,7 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1
 			Allowed: true,
 		}
 	}
+
 	rname := engine.ParseNameFromObject(request.Object.Raw)
 	rns := engine.ParseNamespaceFromObject(request.Object.Raw)
 	rkind := engine.ParseKindFromObject(request.Object.Raw)
@@ -64,18 +64,18 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1
 				glog.Warning(r.Msgs)
 			}
 		} else {
-			// CleanUp Violations if exists
-			err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Validation)
-			if err != nil {
-				glog.Info(err)
-			}
+			// // CleanUp Violations if exists
+			// err := ws.violationBuilder.RemoveInactiveViolation(policy.Name, request.Kind.Kind, rns, rname, info.Validation)
+			// if err != nil {
+			// 	glog.Info(err)
+			// }
 
 			if len(ruleInfos) > 0 {
 				glog.Infof("Validation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, rname, rns)
 			}
 		}
 		policyInfos = append(policyInfos, policyInfo)
-		annPatch := addAnnotationsToResource(request.Object.Raw, policyInfo, info.Mutation)
+		annPatch := addAnnotationsToResource(request.Object.Raw, policyInfo, info.Validation)
 		if annPatch != nil {
 			if annPatches == nil {
 				annPatches = annPatch
@@ -89,13 +89,13 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest) *v1
 	}
 
 	if len(policyInfos) > 0 && len(policyInfos[0].Rules) != 0 {
-		eventsInfo, violations = newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update), true)
+		eventsInfo, _ = newEventInfoFromPolicyInfo(policyInfos, (request.Operation == v1beta1.Update), true)
 		// If the validationFailureAction flag is set "report",
 		// then we dont block the request and report the violations
-		ws.violationBuilder.Add(violations...)
+		// ws.violationBuilder.Add(violations...)
 		ws.eventController.Add(eventsInfo...)
 	}
-	// add annotations
+	//	add annotations
 	if annPatches != nil {
 		ws.annotationsController.Add(rkind, rns, rname, annPatches)
 	}