mirrors/kyverno
mirrors/kyverno
1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-03-31 03:45:17 +00:00
Code Activity

refactor cluster and oplicy violation cleanup

Browse source
This commit is contained in:
shivkumar dudhani 2019-11-27 11:23:29 -08:00
parent 2476940ddf
commit e7607fae87
5 changed files with 248 additions and 797 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
pkg/policy/cleanup.go
View file

@ -21,7 +21,7 @@ func (pc *PolicyController) cleanUpPolicyViolation(pResponse engine.PolicyRespon
// there can be multiple violations as a resource can have multiple owners
if pResponse.Resource.Namespace == "" {
pvs, err := getClusterPVs(pc.pvLister, pc.client, pResponse.Policy, pResponse.Resource.Kind, pResponse.Resource.Name)
pvs, err := getClusterPVs(pc.cpvLister, pc.client, pResponse.Policy, pResponse.Resource.Kind, pResponse.Resource.Name)
if err != nil {
glog.Errorf("failed to cleanUp violations: %v", err)
return
@ -32,7 +32,7 @@ func (pc *PolicyController) cleanUpPolicyViolation(pResponse engine.PolicyRespon
continue
}
glog.V(4).Infof("cleanup cluster violation %s on %s", pv.Name, pv.Spec.ResourceSpec.ToKey())
if err := pc.pvControl.DeletePolicyViolation(pv.Name); err != nil {
if err := pc.pvControl.DeleteClusterPolicyViolation(pv.Name); err != nil {
glog.Errorf("failed to delete cluster policy violation %s on %s: %v", pv.Name, pv.Spec.ResourceSpec.ToKey(), err)
continue
}

132
pkg/policy/clusterpv.go Normal file
View file

@ -0,0 +1,132 @@
package policy
import (
"fmt"
"github.com/golang/glog"
kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1"
"k8s.io/client-go/tools/cache"
)
func (pc *PolicyController) addClusterPolicyViolation(obj interface{}) {
pv := obj.(*kyverno.ClusterPolicyViolation)
if pv.DeletionTimestamp != nil {
// On a restart of the controller manager, it's possible for an object to
// show up in a state that is already pending deletion.
pc.deleteClusterPolicyViolation(pv)
return
}
// dont manage controller references as the ownerReference is assigned by violation generator
ps := pc.getPolicyForClusterPolicyViolation(pv)
if len(ps) == 0 {
// there is no cluster policy for this violation, so we can delete this cluster policy violation
glog.V(4).Infof("Cluster Policy Violation %s does not belong to an active policy, will be cleanedup", pv.Name)
if err := pc.pvControl.DeleteClusterPolicyViolation(pv.Name); err != nil {
glog.Errorf("Failed to deleted cluster policy violation %s: %v", pv.Name, err)
return
}
glog.V(4).Infof("Cluster Policy Violation %s deleted", pv.Name)
return
}
glog.V(4).Infof("Cluster Policy Violation %s added.", pv.Name)
for _, p := range ps {
pc.enqueuePolicy(p)
}
}
func (pc *PolicyController) updateClusterPolicyViolation(old, cur interface{}) {
curPV := cur.(*kyverno.ClusterPolicyViolation)
oldPV := old.(*kyverno.ClusterPolicyViolation)
if curPV.ResourceVersion == oldPV.ResourceVersion {
// Periodic resync will send update events for all known Policy Violation.
// Two different versions of the same replica set will always have different RVs.
return
}
ps := pc.getPolicyForClusterPolicyViolation(curPV)
if len(ps) == 0 {
// there is no cluster policy for this violation, so we can delete this cluster policy violation
glog.V(4).Infof("Cluster Policy Violation %s does not belong to an active policy, will be cleanedup", curPV.Name)
if err := pc.pvControl.DeleteClusterPolicyViolation(curPV.Name); err != nil {
glog.Errorf("Failed to deleted cluster policy violation %s: %v", curPV.Name, err)
return
}
glog.V(4).Infof("PolicyViolation %s deleted", curPV.Name)
return
}
glog.V(4).Infof("Cluster PolicyViolation %s updated", curPV.Name)
for _, p := range ps {
pc.enqueuePolicy(p)
}
}
// deletePolicyViolation enqueues the Policy that manages a PolicyViolation when
// the PolicyViolation is deleted. obj could be an *kyverno.CusterPolicyViolation, or
// a DeletionFinalStateUnknown marker item.
func (pc *PolicyController) deleteClusterPolicyViolation(obj interface{}) {
pv, ok := obj.(*kyverno.ClusterPolicyViolation)
// When a delete is dropped, the relist will notice a PolicyViolation in the store not
// in the list, leading to the insertion of a tombstone object which contains
// the deleted key/value. Note that this value might be stale. If the PolicyViolation
// changed labels the new Policy will not be woken up till the periodic resync.
if !ok {
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
if !ok {
glog.Info(fmt.Errorf("Couldn't get object from tombstone %#v", obj))
return
}
pv, ok = tombstone.Obj.(*kyverno.ClusterPolicyViolation)
if !ok {
glog.Info(fmt.Errorf("Couldn't get object from tombstone %#v", obj))
return
}
}
ps := pc.getPolicyForClusterPolicyViolation(pv)
if len(ps) == 0 {
// there is no cluster policy for this violation, so we can delete this cluster policy violation
glog.V(4).Infof("Cluster Policy Violation %s does not belong to an active policy, will be cleanedup", pv.Name)
if err := pc.pvControl.DeleteClusterPolicyViolation(pv.Name); err != nil {
glog.Errorf("Failed to deleted cluster policy violation %s: %v", pv.Name, err)
return
}
glog.V(4).Infof("Cluster Policy Violation %s deleted", pv.Name)
return
}
glog.V(4).Infof("Cluster PolicyViolation %s updated", pv.Name)
for _, p := range ps {
pc.enqueuePolicy(p)
}
}
func (pc *PolicyController) getPolicyForClusterPolicyViolation(pv *kyverno.ClusterPolicyViolation) []*kyverno.ClusterPolicy {
policies, err := pc.pLister.GetPolicyForPolicyViolation(pv)
if err != nil || len(policies) == 0 {
return nil
}
// Because all PolicyViolations's belonging to a Policy should have a unique label key,
// there should never be more than one Policy returned by the above method.
// If that happens we should probably dynamically repair the situation by ultimately
// trying to clean up one of the controllers, for now we just return the older one
if len(policies) > 1 {
// ControllerRef will ensure we don't do anything crazy, but more than one
// item in this list nevertheless constitutes user error.
glog.V(4).Infof("user error! more than one policy is selecting policy violation %s with labels: %#v, returning %s",
pv.Name, pv.Labels, policies[0].Name)
}
return policies
}
func (pc *PolicyController) getClusterPolicyViolationForPolicy(policy *kyverno.ClusterPolicy) ([]*kyverno.ClusterPolicyViolation, error) {
policySelector, err := buildPolicyLabel(policy.Name)
if err != nil {
return nil, err
}
// Get List of cluster policy violation
cpvList, err := pc.cpvLister.List(policySelector)
if err != nil {
return nil, err
}
return cpvList, nil
}

23
pkg/policy/common.go Normal file
View file

@ -0,0 +1,23 @@
package policy
import (
"fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
)
func buildPolicyLabel(policyName string) (labels.Selector, error) {
policyLabelmap := map[string]string{"policy": policyName}
//NOt using a field selector, as the match function will have to cast the runtime.object
// to get the field, while it can get labels directly, saves the cast effort
ls := &metav1.LabelSelector{}
if err := metav1.Convert_Map_string_To_string_To_v1_LabelSelector(&policyLabelmap, ls, nil); err != nil {
return nil, fmt.Errorf("failed to generate label sector of Policy name %s: %v", policyName, err)
}
policySelector, err := metav1.LabelSelectorAsSelector(ls)
if err != nil {
return nil, fmt.Errorf("Policy %s has invalid label selector: %v", policyName, err)
}
return policySelector, nil
}

753
pkg/policy/controller.go
View file

@ -1,10 +1,8 @@
package policy
import (
"encoding/json"
"fmt"
"reflect"
"sync"
"time"
"github.com/golang/glog"
@ -22,10 +20,6 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
mconfiginformer "k8s.io/client-go/informers/admissionregistration/v1beta1"
@ -64,13 +58,13 @@ type PolicyController struct {
// pLister can list/get policy from the shared informer's store
pLister kyvernolister.ClusterPolicyLister
// pvLister can list/get policy violation from the shared informer's store
pvLister kyvernolister.ClusterPolicyViolationLister
cpvLister kyvernolister.ClusterPolicyViolationLister
// nspvLister can list/get namespaced policy violation from the shared informer's store
nspvLister kyvernolister.NamespacedPolicyViolationLister
// pListerSynced returns true if the Policy store has been synced at least once
pListerSynced cache.InformerSynced
// pvListerSynced returns true if the Policy store has been synced at least once
pvListerSynced cache.InformerSynced
cpvListerSynced cache.InformerSynced
// pvListerSynced returns true if the Policy Violation store has been synced at least once
nspvListerSynced cache.InformerSynced
// mwebhookconfigSynced returns true if the Mutating Webhook Config store has been synced at least once
@ -95,7 +89,7 @@ type PolicyController struct {
func NewPolicyController(kyvernoClient *kyvernoclient.Clientset,
client *client.Client,
pInformer kyvernoinformer.ClusterPolicyInformer,
pvInformer kyvernoinformer.ClusterPolicyViolationInformer,
cpvInformer kyvernoinformer.ClusterPolicyViolationInformer,
nspvInformer kyvernoinformer.NamespacedPolicyViolationInformer,
mconfigwebhookinformer mconfiginformer.MutatingWebhookConfigurationInformer,
webhookRegistrationClient *webhookconfig.WebhookRegistrationClient,
@ -132,10 +126,10 @@ func NewPolicyController(kyvernoClient *kyvernoclient.Clientset,
DeleteFunc: pc.deletePolicy,
})
pvInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: pc.addPolicyViolation,
UpdateFunc: pc.updatePolicyViolation,
DeleteFunc: pc.deletePolicyViolation,
cpvInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: pc.addClusterPolicyViolation,
UpdateFunc: pc.updateClusterPolicyViolation,
DeleteFunc: pc.deleteClusterPolicyViolation,
})
nspvInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
@ -148,11 +142,11 @@ func NewPolicyController(kyvernoClient *kyvernoclient.Clientset,
pc.syncHandler = pc.syncPolicy
pc.pLister = pInformer.Lister()
pc.pvLister = pvInformer.Lister()
pc.cpvLister = cpvInformer.Lister()
pc.nspvLister = nspvInformer.Lister()
pc.pListerSynced = pInformer.Informer().HasSynced
pc.pvListerSynced = pvInformer.Informer().HasSynced
pc.cpvListerSynced = cpvInformer.Informer().HasSynced
pc.nspvListerSynced = nspvInformer.Informer().HasSynced
pc.mwebhookconfigSynced = mconfigwebhookinformer.Informer().HasSynced
pc.mWebhookConfigLister = mconfigwebhookinformer.Lister()
@ -207,181 +201,6 @@ func (pc *PolicyController) deletePolicy(obj interface{}) {
pc.enqueuePolicy(p)
}
func (pc *PolicyController) addPolicyViolation(obj interface{}) {
pv := obj.(*kyverno.ClusterPolicyViolation)
if pv.DeletionTimestamp != nil {
// On a restart of the controller manager, it's possible for an object to
// show up in a state that is already pending deletion.
pc.deletePolicyViolation(pv)
return
}
// generate labels to match the policy from the spec, if not present
if updatePolicyLabelIfNotDefined(pc.pvControl, pv) {
return
}
// If it has a ControllerRef, that's all that matters.
if controllerRef := metav1.GetControllerOf(pv); controllerRef != nil {
p := pc.resolveControllerRef(controllerRef)
if p == nil {
return
}
glog.V(4).Infof("PolicyViolation %s added.", pv.Name)
pc.enqueuePolicy(p)
return
}
// Otherwise, it's an orphan. Get a list of all matching Policies and sync
// them to see if anyone wants to adopt it.
ps := pc.getPolicyForPolicyViolation(pv)
if len(ps) == 0 {
// there is no cluster policy for this violation, so we can delete this cluster policy violation
glog.V(4).Infof("PolicyViolation %s does not belong to an active policy, will be cleanedup", pv.Name)
if err := pc.pvControl.DeletePolicyViolation(pv.Name); err != nil {
glog.Errorf("Failed to deleted policy violation %s: %v", pv.Name, err)
return
}
glog.V(4).Infof("PolicyViolation %s deleted", pv.Name)
return
}
glog.V(4).Infof("Orphan Policy Violation %s added.", pv.Name)
for _, p := range ps {
pc.enqueuePolicy(p)
}
}
func (pc *PolicyController) updatePolicyViolation(old, cur interface{}) {
curPV := cur.(*kyverno.ClusterPolicyViolation)
oldPV := old.(*kyverno.ClusterPolicyViolation)
if curPV.ResourceVersion == oldPV.ResourceVersion {
// Periodic resync will send update events for all known Policy Violation.
// Two different versions of the same replica set will always have different RVs.
return
}
// generate labels to match the policy from the spec, if not present
if updatePolicyLabelIfNotDefined(pc.pvControl, curPV) {
return
}
curControllerRef := metav1.GetControllerOf(curPV)
oldControllerRef := metav1.GetControllerOf(oldPV)
controllerRefChanged := !reflect.DeepEqual(curControllerRef, oldControllerRef)
if controllerRefChanged && oldControllerRef != nil {
// The ControllerRef was changed. Sync the old controller, if any.
if p := pc.resolveControllerRef(oldControllerRef); p != nil {
pc.enqueuePolicy(p)
}
}
// If it has a ControllerRef, that's all that matters.
if curControllerRef != nil {
p := pc.resolveControllerRef(curControllerRef)
if p == nil {
return
}
glog.V(4).Infof("PolicyViolation %s updated.", curPV.Name)
pc.enqueuePolicy(p)
return
}
// Otherwise, it's an orphan. If anything changed, sync matching controllers
// to see if anyone wants to adopt it now.
labelChanged := !reflect.DeepEqual(curPV.Labels, oldPV.Labels)
if labelChanged || controllerRefChanged {
ps := pc.getPolicyForPolicyViolation(curPV)
if len(ps) == 0 {
// there is no cluster policy for this violation, so we can delete this cluster policy violation
glog.V(4).Infof("PolicyViolation %s does not belong to an active policy, will be cleanedup", curPV.Name)
if err := pc.pvControl.DeletePolicyViolation(curPV.Name); err != nil {
glog.Errorf("Failed to deleted policy violation %s: %v", curPV.Name, err)
return
}
glog.V(4).Infof("PolicyViolation %s deleted", curPV.Name)
return
}
glog.V(4).Infof("Orphan PolicyViolation %s updated", curPV.Name)
for _, p := range ps {
pc.enqueuePolicy(p)
}
}
}
// deletePolicyViolation enqueues the Policy that manages a PolicyViolation when
// the PolicyViolation is deleted. obj could be an *kyverno.CusterPolicyViolation, or
// a DeletionFinalStateUnknown marker item.
func (pc *PolicyController) deletePolicyViolation(obj interface{}) {
pv, ok := obj.(*kyverno.ClusterPolicyViolation)
// When a delete is dropped, the relist will notice a PolicyViolation in the store not
// in the list, leading to the insertion of a tombstone object which contains
// the deleted key/value. Note that this value might be stale. If the PolicyViolation
// changed labels the new Policy will not be woken up till the periodic resync.
if !ok {
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
if !ok {
glog.Info(fmt.Errorf("Couldn't get object from tombstone %#v", obj))
return
}
pv, ok = tombstone.Obj.(*kyverno.ClusterPolicyViolation)
if !ok {
glog.Info(fmt.Errorf("Couldn't get object from tombstone %#v", obj))
return
}
}
controllerRef := metav1.GetControllerOf(pv)
if controllerRef == nil {
// No controller should care about orphans being deleted.
return
}
p := pc.resolveControllerRef(controllerRef)
if p == nil {
return
}
glog.V(4).Infof("PolicyViolation %s deleted", pv.Name)
pc.enqueuePolicy(p)
}
// resolveControllerRef returns the controller referenced by a ControllerRef,
// or nil if the ControllerRef could not be resolved to a matching controller
// of the correct Kind.
func (pc *PolicyController) resolveControllerRef(controllerRef *metav1.OwnerReference) *kyverno.ClusterPolicy {
// We can't look up by UID, so look up by Name and then verify UID.
// Don't even try to look up by Name if it's the wrong Kind.
if controllerRef.Kind != controllerRef.Kind {
return nil
}
p, err := pc.pLister.Get(controllerRef.Name)
if err != nil {
return nil
}
if p.UID != controllerRef.UID {
// The controller we found with this Name is not the same one that the
// ControllerRef points to.
return nil
}
return p
}
func (pc *PolicyController) getPolicyForPolicyViolation(pv *kyverno.ClusterPolicyViolation) []*kyverno.ClusterPolicy {
policies, err := pc.pLister.GetPolicyForPolicyViolation(pv)
if err != nil || len(policies) == 0 {
return nil
}
// Because all PolicyViolations's belonging to a Policy should have a unique label key,
// there should never be more than one Policy returned by the above method.
// If that happens we should probably dynamically repair the situation by ultimately
// trying to clean up one of the controllers, for now we just return the older one
if len(policies) > 1 {
// ControllerRef will ensure we don't do anything crazy, but more than one
// item in this list nevertheless constitutes user error.
glog.V(4).Infof("user error! more than one policy is selecting policy violation %s with labels: %#v, returning %s",
pv.Name, pv.Labels, policies[0].Name)
}
return policies
}
func (pc *PolicyController) enqueue(policy *kyverno.ClusterPolicy) {
key, err := cache.MetaNamespaceKeyFunc(policy)
if err != nil {
@ -400,7 +219,7 @@ func (pc *PolicyController) Run(workers int, stopCh <-chan struct{}) {
glog.Info("Starting policy controller")
defer glog.Info("Shutting down policy controller")
if !cache.WaitForCacheSync(stopCh, pc.pListerSynced, pc.pvListerSynced, pc.nspvListerSynced, pc.mwebhookconfigSynced) {
if !cache.WaitForCacheSync(stopCh, pc.pListerSynced, pc.cpvListerSynced, pc.nspvListerSynced, pc.mwebhookconfigSynced) {
glog.Error("failed to sync informer cache")
return
}
@ -458,9 +277,17 @@ func (pc *PolicyController) syncPolicy(key string) error {
policy, err := pc.pLister.Get(key)
if errors.IsNotFound(err) {
glog.V(2).Infof("Policy %v has been deleted", key)
// delete cluster policy violation
if err := pc.deleteClusterPolicyViolations(policy); err != nil {
return err
}
// delete namespaced policy violation
if err := pc.deleteNamespacedPolicyViolations(policy); err != nil {
return err
}
// remove the recorded stats for the policy
pc.statusAggregator.RemovePolicyStats(key)
// remove webhook configurations if there are no policies
if err := pc.removeResourceWebhookConfiguration(); err != nil {
// do not fail, if unable to delete resource webhook config
@ -469,9 +296,7 @@ func (pc *PolicyController) syncPolicy(key string) error {
}
return nil
}
if err != nil {
glog.V(4).Info(err)
return err
}
@ -480,25 +305,52 @@ func (pc *PolicyController) syncPolicy(key string) error {
glog.Errorln(err)
}
// Deep-copy otherwise we are mutating our cache.
// TODO: Deep-copy only when needed.
p := policy.DeepCopy()
pvList, nspvList, err := pc.getPolicyViolationsForPolicy(p)
// cluster policy violations
cpvList, err := pc.getClusterPolicyViolationForPolicy(policy)
if err != nil {
return err
}
// namespaced policy violation
nspvList, err := pc.getNamespacedPolicyViolationForPolicy(policy)
if err != nil {
return err
}
// process policies on existing resources
engineResponses := pc.processExistingResources(*p)
engineResponses := pc.processExistingResources(*policy)
// report errors
pc.cleanupAndReport(engineResponses)
// sync active
return pc.syncStatusOnly(p, pvList, nspvList)
return pc.syncStatusOnly(policy, cpvList, nspvList)
}
func (pc *PolicyController) deleteClusterPolicyViolations(policy *kyverno.ClusterPolicy) error {
cpvList, err := pc.getClusterPolicyViolationForPolicy(policy)
if err != nil {
return err
}
for _, cpv := range cpvList {
if err := pc.pvControl.DeleteClusterPolicyViolation(cpv.Name); err != nil {
return err
}
}
return nil
}
func (pc *PolicyController) deleteNamespacedPolicyViolations(policy *kyverno.ClusterPolicy) error {
nspvList, err := pc.getNamespacedPolicyViolationForPolicy(policy)
if err != nil {
return err
}
for _, nspv := range nspvList {
if err := pc.pvControl.DeleteNamespacedPolicyViolation(nspv.Namespace, nspv.Name); err != nil {
return err
}
}
return nil
}
//syncStatusOnly updates the policy status subresource
// status:
// - violations : (count of the resources that violate this policy )
func (pc *PolicyController) syncStatusOnly(p *kyverno.ClusterPolicy, pvList []*kyverno.ClusterPolicyViolation, nspvList []*kyverno.NamespacedPolicyViolation) error {
newStatus := pc.calculateStatus(p.Name, pvList, nspvList)
if reflect.DeepEqual(newStatus, p.Status) {
@ -531,346 +383,23 @@ func (pc *PolicyController) calculateStatus(policyName string, pvList []*kyverno
return status
}
func (pc *PolicyController) getPolicyViolationsForPolicy(p *kyverno.ClusterPolicy) ([]*kyverno.ClusterPolicyViolation, []*kyverno.NamespacedPolicyViolation, error) {
policyLabelmap := map[string]string{"policy": p.Name}
//NOt using a field selector, as the match function will have to cash the runtime.object
// to get the field, while it can get labels directly, saves the cast effort
ls := &metav1.LabelSelector{}
if err := metav1.Convert_Map_string_To_string_To_v1_LabelSelector(&policyLabelmap, ls, nil); err != nil {
return nil, nil, fmt.Errorf("failed to generate label sector of Policy name %s: %v", p.Name, err)
}
policySelector, err := metav1.LabelSelectorAsSelector(ls)
func (pc *PolicyController) getNamespacedPolicyViolationForPolicy(policy *kyverno.ClusterPolicy) ([]*kyverno.NamespacedPolicyViolation, error) {
policySelector, err := buildPolicyLabel(policy.Name)
if err != nil {
return nil, nil, fmt.Errorf("Policy %s has invalid label selector: %v", p.Name, err)
return nil, err
}
// List all PolicyViolation to find those we own but that no longer match our
// selector. They will be orphaned by ClaimPolicyViolation().
// cluster Policy Violation
pvList, err := pc.pvLister.List(labels.Everything())
// Get List of cluster policy violation
nspvList, err := pc.nspvLister.List(policySelector)
if err != nil {
return nil, nil, err
return nil, err
}
canAdoptFunc := RecheckDeletionTimestamp(func() (metav1.Object, error) {
fresh, err := pc.kyvernoClient.KyvernoV1().ClusterPolicies().Get(p.Name, metav1.GetOptions{})
if err != nil {
return nil, err
}
if fresh.UID != p.UID {
return nil, fmt.Errorf("original Policy %v is gone: got uid %v, wanted %v", p.Name, fresh.UID, p.UID)
}
return fresh, nil
})
cm := NewPolicyViolationControllerRefManager(pc.pvControl, p, policySelector, controllerKind, canAdoptFunc)
claimedPVList, err := cm.claimPolicyViolations(pvList)
if err != nil {
return nil, nil, err
}
claimedPVs := claimedPVList.([]*kyverno.ClusterPolicyViolation)
// namespaced Policy Violation
nspvList, err := pc.nspvLister.List(labels.Everything())
if err != nil {
return nil, nil, err
}
nscm := NewPolicyViolationControllerRefManager(pc.pvControl, p, policySelector, controllerKind, canAdoptFunc)
claimedNSPVList, err := nscm.claimPolicyViolations(nspvList)
if err != nil {
return nil, nil, err
}
claimedNSPVs := claimedNSPVList.([]*kyverno.NamespacedPolicyViolation)
return claimedPVs, claimedNSPVs, nil
}
func (m *PolicyViolationControllerRefManager) claimPolicyViolations(sets interface{}) (interface{}, error) {
var errlist []error
match := func(obj metav1.Object) bool {
return m.Selector.Matches(labels.Set(obj.GetLabels()))
}
adopt := func(obj metav1.Object) error {
return m.adoptPolicyViolation(obj)
}
release := func(obj metav1.Object) error {
return m.releasePolicyViolation(obj)
}
if pvs, ok := sets.([]*kyverno.ClusterPolicyViolation); ok {
var claimed []*kyverno.ClusterPolicyViolation
for _, pv := range pvs {
ok, err := m.ClaimObject(pv, match, adopt, release)
if err != nil {
errlist = append(errlist, err)
continue
}
if ok {
claimed = append(claimed, pv)
}
}
return claimed, utilerrors.NewAggregate(errlist)
}
var claimed []*kyverno.NamespacedPolicyViolation
for _, pv := range sets.([]*kyverno.NamespacedPolicyViolation) {
ok, err := m.ClaimObject(pv, match, adopt, release)
if err != nil {
errlist = append(errlist, err)
continue
}
if ok {
claimed = append(claimed, pv)
}
}
return claimed, utilerrors.NewAggregate(errlist)
}
func (m *PolicyViolationControllerRefManager) adoptPolicyViolation(pv interface{}) error {
var ns, pvname string
var pvuid types.UID
switch typedPV := pv.(type) {
case *kyverno.ClusterPolicyViolation:
pvname = typedPV.Name
pvuid = typedPV.UID
case *kyverno.NamespacedPolicyViolation:
ns = typedPV.Namespace
pvname = typedPV.Name
pvuid = typedPV.UID
}
if err := m.CanAdopt(); err != nil {
return fmt.Errorf("can't adopt %T name=%s, namespace=%s (%v): %v", pv, pvname, ns, pvuid, err)
}
// Note that ValidateOwnerReferences() will reject this patch if another
// OwnerReference exists with controller=true.
//TODO Add JSON Patch Owner reference for resource
//TODO Update owner refence for resource
controllerFlag := true
blockOwnerDeletionFlag := true
pOwnerRef := metav1.OwnerReference{APIVersion: m.controllerKind.GroupVersion().String(),
Kind: m.controllerKind.Kind,
Name: m.Controller.GetName(),
UID: m.Controller.GetUID(),
Controller: &controllerFlag,
BlockOwnerDeletion: &blockOwnerDeletionFlag,
}
addControllerPatch, err := createOwnerReferencePatch(pOwnerRef)
if err != nil {
glog.Errorf("failed to add owner reference %v for PolicyViolation %s: %v", pOwnerRef, pvname, err)
return err
}
if _, ok := pv.(*kyverno.ClusterPolicyViolation); ok {
return m.pvControl.PatchPolicyViolation(pvname, addControllerPatch)
}
return m.pvControl.PatchNamespacedPolicyViolation(ns, pvname, addControllerPatch)
}
type patchOwnerReferenceValue struct {
Op string `json:"op"`
Path string `json:"path"`
Value []metav1.OwnerReference `json:"value"`
}
func createOwnerReferencePatch(ownerRef metav1.OwnerReference) ([]byte, error) {
payload := []patchOwnerReferenceValue{{
Op: "add",
Path: "/metadata/ownerReferences",
Value: []metav1.OwnerReference{ownerRef},
}}
return json.Marshal(payload)
}
func removeOwnerReferencePatch(ownerRef metav1.OwnerReference) ([]byte, error) {
payload := []patchOwnerReferenceValue{{
Op: "remove",
Path: "/metadata/ownerReferences",
Value: []metav1.OwnerReference{ownerRef},
}}
return json.Marshal(payload)
}
func (m *PolicyViolationControllerRefManager) releasePolicyViolation(pv interface{}) error {
var ns, pvname string
switch typedPV := pv.(type) {
case *kyverno.ClusterPolicyViolation:
pvname = typedPV.Name
case *kyverno.NamespacedPolicyViolation:
ns = typedPV.Namespace
pvname = typedPV.Name
}
glog.V(2).Infof("patching PolicyViolation %s to remove its controllerRef to %s/%s:%s",
pvname, m.controllerKind.GroupVersion(), m.controllerKind.Kind, m.Controller.GetName())
//TODO JSON patch for owner reference for resources
controllerFlag := true
blockOwnerDeletionFlag := true
pOwnerRef := metav1.OwnerReference{APIVersion: m.controllerKind.GroupVersion().String(),
Kind: m.controllerKind.Kind,
Name: m.Controller.GetName(),
UID: m.Controller.GetUID(),
Controller: &controllerFlag,
BlockOwnerDeletion: &blockOwnerDeletionFlag,
}
removeControllerPatch, err := removeOwnerReferencePatch(pOwnerRef)
if err != nil {
glog.Errorf("failed to add owner reference %v for PolicyViolation %s: %v", pOwnerRef, pvname, err)
return err
}
// deleteOwnerRefPatch := fmt.Sprintf(`{"metadata":{"ownerReferences":[{"$patch":"delete","uid":"%s"}],"uid":"%s"}}`, m.Controller.GetUID(), pv.UID)
if _, ok := pv.(*kyverno.ClusterPolicyViolation); ok {
err = m.pvControl.PatchPolicyViolation(pvname, removeControllerPatch)
if err != nil {
if errors.IsNotFound(err) {
// If the ReplicaSet no longer exists, ignore it.
return nil
}
if errors.IsInvalid(err) {
// Invalid error will be returned in two cases: 1. the ReplicaSet
// has no owner reference, 2. the uid of the ReplicaSet doesn't
// match, which means the ReplicaSet is deleted and then recreated.
// In both cases, the error can be ignored.
return nil
}
}
return err
}
err = m.pvControl.PatchNamespacedPolicyViolation(ns, pvname, removeControllerPatch)
if err != nil {
if errors.IsNotFound(err) {
// If the ReplicaSet no longer exists, ignore it.
return nil
}
if errors.IsInvalid(err) {
// Invalid error will be returned in two cases: 1. the ReplicaSet
// has no owner reference, 2. the uid of the ReplicaSet doesn't
// match, which means the ReplicaSet is deleted and then recreated.
// In both cases, the error can be ignored.
return nil
}
}
return err
}
//PolicyViolationControllerRefManager manages adoption of policy violation by a policy
type PolicyViolationControllerRefManager struct {
BaseControllerRefManager
controllerKind schema.GroupVersionKind
pvControl PVControlInterface
}
//NewPolicyViolationControllerRefManager returns new PolicyViolationControllerRefManager
func NewPolicyViolationControllerRefManager(
pvControl PVControlInterface,
controller metav1.Object,
selector labels.Selector,
controllerKind schema.GroupVersionKind,
canAdopt func() error,
) *PolicyViolationControllerRefManager {
m := PolicyViolationControllerRefManager{
BaseControllerRefManager: BaseControllerRefManager{
Controller: controller,
Selector: selector,
CanAdoptFunc: canAdopt,
},
controllerKind: controllerKind,
pvControl: pvControl,
}
return &m
}
//BaseControllerRefManager ...
type BaseControllerRefManager struct {
Controller metav1.Object
Selector labels.Selector
canAdoptErr error
canAdoptOnce sync.Once
CanAdoptFunc func() error
}
//CanAdopt ...
func (m *BaseControllerRefManager) CanAdopt() error {
m.canAdoptOnce.Do(func() {
if m.CanAdoptFunc != nil {
m.canAdoptErr = m.CanAdoptFunc()
}
})
return m.canAdoptErr
}
//ClaimObject ...
func (m *BaseControllerRefManager) ClaimObject(obj metav1.Object, match func(metav1.Object) bool, adopt, release func(metav1.Object) error) (bool, error) {
controllerRef := metav1.GetControllerOf(obj)
if controllerRef != nil {
if controllerRef.UID != m.Controller.GetUID() {
// Owned by someone else. Ignore
return false, nil
}
if match(obj) {
// We already own it and the selector matches.
// Return true (successfully claimed) before checking deletion timestamp.
// We're still allowed to claim things we already own while being deleted
// because doing so requires taking no actions.
return true, nil
}
// Owned by us but selector doesn't match.
// Try to release, unless we're being deleted.
if m.Controller.GetDeletionTimestamp() != nil {
return false, nil
}
if err := release(obj); err != nil {
// If the PolicyViolation no longer exists, ignore the error.
if errors.IsNotFound(err) {
return false, nil
}
// Either someone else released it, or there was a transient error.
// The controller should requeue and try again if it's still stale.
return false, err
}
// Successfully released.
return false, nil
}
// It's an orphan.
if m.Controller.GetDeletionTimestamp() != nil || !match(obj) {
// Ignore if we're being deleted or selector doesn't match.
return false, nil
}
if obj.GetDeletionTimestamp() != nil {
// Ignore if the object is being deleted
return false, nil
}
// Selector matches. Try to adopt.
if err := adopt(obj); err != nil {
// If the PolicyViolation no longer exists, ignore the error
if errors.IsNotFound(err) {
return false, nil
}
// Either someone else claimed it first, or there was a transient error.
// The controller should requeue and try again if it's still orphaned.
return false, err
}
// Successfully adopted.
return true, nil
return nspvList, nil
}
//PVControlInterface provides interface to operate on policy violation resource
type PVControlInterface interface {
PatchPolicyViolation(name string, data []byte) error
DeletePolicyViolation(name string) error
PatchNamespacedPolicyViolation(ns, name string, data []byte) error
DeleteClusterPolicyViolation(name string) error
DeleteNamespacedPolicyViolation(ns, name string) error
}
@ -880,170 +409,16 @@ type RealPVControl struct {
Recorder record.EventRecorder
}
//PatchPolicyViolation patches the policy violation with the provided JSON Patch
func (r RealPVControl) PatchPolicyViolation(name string, data []byte) error {
_, err := r.Client.KyvernoV1().ClusterPolicyViolations().Patch(name, types.JSONPatchType, data)
return err
}
//DeletePolicyViolation deletes the policy violation
func (r RealPVControl) DeletePolicyViolation(name string) error {
func (r RealPVControl) DeleteClusterPolicyViolation(name string) error {
return r.Client.KyvernoV1().ClusterPolicyViolations().Delete(name, &metav1.DeleteOptions{})
}
//PatchNamespacedPolicyViolation patches the namespaced policy violation with the provided JSON Patch
func (r RealPVControl) PatchNamespacedPolicyViolation(ns, name string, data []byte) error {
_, err := r.Client.KyvernoV1().NamespacedPolicyViolations(ns).Patch(name, types.JSONPatchType, data)
return err
}
//DeleteNamespacedPolicyViolation deletes the namespaced policy violation
func (r RealPVControl) DeleteNamespacedPolicyViolation(ns, name string) error {
return r.Client.KyvernoV1().NamespacedPolicyViolations(ns).Delete(name, &metav1.DeleteOptions{})
}
// RecheckDeletionTimestamp returns a CanAdopt() function to recheck deletion.
//
// The CanAdopt() function calls getObject() to fetch the latest value,
// and denies adoption attempts if that object has a non-nil DeletionTimestamp.
func RecheckDeletionTimestamp(getObject func() (metav1.Object, error)) func() error {
return func() error {
obj, err := getObject()
if err != nil {
return fmt.Errorf("can't recheck DeletionTimestamp: %v", err)
}
if obj.GetDeletionTimestamp() != nil {
return fmt.Errorf("%v/%v has just been deleted at %v", obj.GetNamespace(), obj.GetName(), obj.GetDeletionTimestamp())
}
return nil
}
}
type patchLabelValue struct {
Op string `json:"op"`
Path string `json:"path"`
Value string `json:"value"`
}
type patchLabelMapValue struct {
Op string `json:"op"`
Path string `json:"path"`
Value map[string]string `json:"value"`
}
func createPolicyLabelPatch(policy string) ([]byte, error) {
payload := []patchLabelValue{{
Op: "add",
Path: "/metadata/labels/policy",
Value: policy,
}}
return json.Marshal(payload)
}
func createResourceLabelPatch(resource string) ([]byte, error) {
payload := []patchLabelValue{{
Op: "add",
Path: "/metadata/labels/resource",
Value: resource,
}}
return json.Marshal(payload)
}
func createLabelMapPatch(policy string, resource string) ([]byte, error) {
payload := []patchLabelMapValue{{
Op: "add",
Path: "/metadata/labels",
Value: map[string]string{"policy": policy, "resource": resource},
}}
return json.Marshal(payload)
}
//updatePolicyLabelIfNotDefined adds the label 'policy' to the PolicyViolation
// label is used here to lookup policyViolation and corresponding Policy
func updatePolicyLabelIfNotDefined(pvControl PVControlInterface, pv *kyverno.ClusterPolicyViolation) bool {
updateLabel := func() bool {
glog.V(4).Infof("adding label 'policy:%s' to PolicyViolation %s", pv.Spec.Policy, pv.Name)
glog.V(4).Infof("adding label 'resource:%s' to PolicyViolation %s", pv.Spec.ResourceSpec.ToKey(), pv.Name)
// add label based on the policy spec
labels := pv.GetLabels()
if pv.Spec.Policy == "" {
glog.Error("policy not defined for violation")
// should be cleaned up
return false
}
if labels == nil {
// create a patch to generate the labels map with policy label
patch, err := createLabelMapPatch(pv.Spec.Policy, pv.Spec.ResourceSpec.ToKey())
if err != nil {
glog.Errorf("unable to init label map. %v", err)
return false
}
if err := pvControl.PatchPolicyViolation(pv.Name, patch); err != nil {
glog.Errorf("Unable to add 'policy' label to PolicyViolation %s: %v", pv.Name, err)
return false
}
// update successful
return true
}
// JSON Patch to add exact label
policyLabelPatch, err := createPolicyLabelPatch(pv.Spec.Policy)
if err != nil {
glog.Errorf("failed to generate patch to add label 'policy': %v", err)
return false
}
resourceLabelPatch, err := createResourceLabelPatch(pv.Spec.ResourceSpec.ToKey())
if err != nil {
glog.Errorf("failed to generate patch to add label 'resource': %v", err)
return false
}
//join patches
labelPatch := joinPatches(policyLabelPatch, resourceLabelPatch)
if labelPatch == nil {
glog.Errorf("failed to join patches : %v", err)
return false
}
glog.V(4).Infof("patching policy violation %s with patch %s", pv.Name, string(labelPatch))
if err := pvControl.PatchPolicyViolation(pv.Name, labelPatch); err != nil {
glog.Errorf("Unable to add 'policy' label to PolicyViolation %s: %v", pv.Name, err)
return false
}
// update successful
return true
}
var policy string
var ok bool
// operate oncopy of resource
curLabels := pv.GetLabels()
if policy, ok = curLabels["policy"]; !ok {
return updateLabel()
}
// TODO: would be benificial to add a check to verify if the policy in name and resource spec match
if policy != pv.Spec.Policy {
glog.Errorf("label 'policy:%s' and spec.policy %s dont match ", policy, pv.Spec.Policy)
//TODO handle this case
return updateLabel()
}
return false
}
func joinPatches(patches ...[]byte) []byte {
var result []byte
if patches == nil {
//nothing tot join
return result
}
result = append(result, []byte("[\n")...)
for index, patch := range patches {
result = append(result, patch...)
if index != len(patches)-1 {
result = append(result, []byte(",\n")...)
}
}
result = append(result, []byte("\n]")...)
return result
}
// convertRules converts the internal rule stats to one used in policy.stats struct
func convertRules(rules []RuleStatinfo) []kyverno.RuleStats {
var stats []kyverno.RuleStats

133
pkg/policy/namespacedpv.go
View file

@ -1,11 +1,8 @@
package policy
import (
"reflect"
"github.com/golang/glog"
kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
cache "k8s.io/client-go/tools/cache"
)
@ -15,33 +12,16 @@ func (pc *PolicyController) addNamespacedPolicyViolation(obj interface{}) {
if pv.DeletionTimestamp != nil {
// On a restart of the controller manager, it's possible for an object to
// show up in a state that is already pending deletion.
pc.deletePolicyViolation(pv)
pc.deleteNamespacedPolicyViolation(pv)
return
}
// dont manage controller references as the ownerReference is assigned by violation generator
// generate labels to match the policy from the spec, if not present
if updateLabels(pv) {
return
}
// If it has a ControllerRef, that's all that matters.
if controllerRef := metav1.GetControllerOf(pv); controllerRef != nil {
p := pc.resolveControllerRef(controllerRef)
if p == nil {
return
}
glog.V(4).Infof("Namespaced policy violation %s added.", pv.Name)
pc.enqueuePolicy(p)
return
}
// Otherwise, it's an orphan. Get a list of all matching Policies and sync
// them to see if anyone wants to adopt it.
ps := pc.getPolicyForNamespacedPolicyViolation(pv)
if len(ps) == 0 {
// there is no cluster policy for this violation, so we can delete this cluster policy violation
glog.V(4).Infof("PolicyViolation %s does not belong to an active policy, will be cleanedup", pv.Name)
if err := pc.pvControl.DeletePolicyViolation(pv.Name); err != nil {
if err := pc.pvControl.DeleteNamespacedPolicyViolation(pv.Namespace, pv.Name); err != nil {
glog.Errorf("Failed to deleted policy violation %s: %v", pv.Name, err)
return
}
@ -63,51 +43,22 @@ func (pc *PolicyController) updateNamespacedPolicyViolation(old, cur interface{}
return
}
// generate labels to match the policy from the spec, if not present
if updateLabels(curPV) {
return
}
curControllerRef := metav1.GetControllerOf(curPV)
oldControllerRef := metav1.GetControllerOf(oldPV)
controllerRefChanged := !reflect.DeepEqual(curControllerRef, oldControllerRef)
if controllerRefChanged && oldControllerRef != nil {
// The ControllerRef was changed. Sync the old controller, if any.
if p := pc.resolveControllerRef(oldControllerRef); p != nil {
pc.enqueuePolicy(p)
}
}
// If it has a ControllerRef, that's all that matters.
if curControllerRef != nil {
p := pc.resolveControllerRef(curControllerRef)
if p == nil {
ps := pc.getPolicyForNamespacedPolicyViolation(curPV)
if len(ps) == 0 {
// there is no namespaced policy for this violation, so we can delete this cluster policy violation
glog.V(4).Infof("Namespaced Policy Violation %s does not belong to an active policy, will be cleanedup", curPV.Name)
if err := pc.pvControl.DeleteNamespacedPolicyViolation(curPV.Namespace, curPV.Name); err != nil {
glog.Errorf("Failed to deleted namespaced policy violation %s: %v", curPV.Name, err)
return
}
glog.V(4).Infof("PolicyViolation %s updated.", curPV.Name)
glog.V(4).Infof("Namespaced Policy Violation %s deleted", curPV.Name)
return
}
glog.V(4).Infof("Namespaced Policy sViolation %s updated", curPV.Name)
for _, p := range ps {
pc.enqueuePolicy(p)
return
}
// Otherwise, it's an orphan. If anything changed, sync matching controllers
// to see if anyone wants to adopt it now.
labelChanged := !reflect.DeepEqual(curPV.Labels, oldPV.Labels)
if labelChanged || controllerRefChanged {
ps := pc.getPolicyForNamespacedPolicyViolation(curPV)
if len(ps) == 0 {
// there is no namespaced policy for this violation, so we can delete this cluster policy violation
glog.V(4).Infof("PolicyViolation %s does not belong to an active policy, will be cleanedup", curPV.Name)
if err := pc.pvControl.DeletePolicyViolation(curPV.Name); err != nil {
glog.Errorf("Failed to deleted policy violation %s: %v", curPV.Name, err)
return
}
glog.V(4).Infof("PolicyViolation %s deleted", curPV.Name)
return
}
glog.V(4).Infof("Orphan PolicyViolation %s updated", curPV.Name)
for _, p := range ps {
pc.enqueuePolicy(p)
}
}
}
func (pc *PolicyController) deleteNamespacedPolicyViolation(obj interface{}) {
@ -128,52 +79,22 @@ func (pc *PolicyController) deleteNamespacedPolicyViolation(obj interface{}) {
return
}
}
controllerRef := metav1.GetControllerOf(pv)
if controllerRef == nil {
// No controller should care about orphans being deleted.
ps := pc.getPolicyForNamespacedPolicyViolation(pv)
if len(ps) == 0 {
// there is no cluster policy for this violation, so we can delete this cluster policy violation
glog.V(4).Infof("Namespaced Policy Violation %s does not belong to an active policy, will be cleanedup", pv.Name)
if err := pc.pvControl.DeleteNamespacedPolicyViolation(pv.Namespace, pv.Name); err != nil {
glog.Errorf("Failed to deleted namespaced policy violation %s: %v", pv.Name, err)
return
}
glog.V(4).Infof("Namespaced Policy Violation %s deleted", pv.Name)
return
}
p := pc.resolveControllerRef(controllerRef)
if p == nil {
return
glog.V(4).Infof("Namespaced PolicyViolation %s updated", pv.Name)
for _, p := range ps {
pc.enqueuePolicy(p)
}
glog.V(4).Infof("PolicyViolation %s deleted", pv.Name)
pc.enqueuePolicy(p)
}
func updateLabels(pv *kyverno.NamespacedPolicyViolation) bool {
if pv.Spec.Policy == "" {
glog.Error("policy not defined for violation")
// should be cleaned up
return false
}
labels := pv.GetLabels()
newLabels := labels
if newLabels == nil {
newLabels = make(map[string]string)
}
policy, ok := newLabels["policy"]
// key 'policy' does not present
// or policy name has changed
if !ok || policy != pv.Spec.Policy {
newLabels["policy"] = pv.Spec.Policy
}
resource, ok := newLabels["resource"]
// key 'resource' does not present
// or resource defined in policy has changed
if !ok || resource != pv.Spec.ResourceSpec.ToKey() {
newLabels["resource"] = pv.Spec.ResourceSpec.ToKey()
}
if !reflect.DeepEqual(labels, newLabels) {
pv.SetLabels(labels)
return true
}
return false
}
func (pc *PolicyController) getPolicyForNamespacedPolicyViolation(pv *kyverno.NamespacedPolicyViolation) []*kyverno.ClusterPolicy {