mirrors/kyverno
mirrors/kyverno
1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-03-28 10:28:36 +00:00
Code Activity

refactor: policy cache (#3919)

Browse source 
* refactor: simplify policy cache

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* refactor: policy cache

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* remove update and add policies map

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* fix: review comments

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

Co-authored-by: Vyankatesh Kudtarkar <vyankateshkd@gmail.com>
This commit is contained in:
Charles-Edouard Brétéché 2022-05-16 09:56:16 +02:00 • committed by GitHub
parent 474223dc5b
commit 70954b9995
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 400 additions and 571 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
api/kyverno/v1/match_resources_types.go
View file

@ -32,7 +32,8 @@ type MatchResources struct {
// GetKinds returns all kinds
func (m *MatchResources) GetKinds() []string {
kinds := m.ResourceDescription.Kinds
var kinds []string
kinds = append(kinds, m.ResourceDescription.Kinds...)
for _, value := range m.All {
kinds = append(kinds, value.ResourceDescription.Kinds...)
}

6
cmd/kyverno/main.go
View file

@ -316,10 +316,10 @@ func main() {
os.Exit(1)
}
pCacheController := policycache.NewPolicyCacheController(kyvernoV1.ClusterPolicies(), kyvernoV1.Policies())
pCacheController := policycache.NewCache(kyvernoV1.ClusterPolicies(), kyvernoV1.Policies())
auditHandler := webhooks.NewValidateAuditHandler(
pCacheController.Cache,
pCacheController,
eventGenerator,
reportReqGen,
kubeInformer.Rbac().V1().RoleBindings(),
@ -421,7 +421,7 @@ func main() {
kubeInformer.Rbac().V1().ClusterRoles(),
kubeInformer.Core().V1().Namespaces(),
eventGenerator,
pCacheController.Cache,
pCacheController,
webhookCfg,
webhookMonitor,
configuration,

148
pkg/policycache/cache.go Normal file
View file

@ -0,0 +1,148 @@
package policycache
import (
"os"
"reflect"
"sync/atomic"
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
kyvernov1informer "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1"
kyvernov1lister "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1"
kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/tools/cache"
)
// Cache get method use for to get policy names and mostly use to test cache testcases
type Cache interface {
// GetPolicies returns all policies that apply to a namespace, including cluster-wide policies
// If the namespace is empty, only cluster-wide policies are returned
GetPolicies(PolicyType, string, string) []kyvernov1.PolicyInterface
// CheckPolicySync wait until the internal policy cache is fully loaded
CheckPolicySync(<-chan struct{})
}
// controller is responsible for synchronizing Policy Cache,
// it embeds a policy informer to handle policy events.
// The cache is synced when a policy is add/update/delete.
// This cache is only used in the admission webhook to fast retrieve
// policies based on types (Mutate/ValidateEnforce/Generate/imageVerify).
type controller struct {
store
cpolLister kyvernov1lister.ClusterPolicyLister
polLister kyvernov1lister.PolicyLister
pCounter int64
}
// NewCache create a new Cache
func NewCache(pInformer kyvernov1informer.ClusterPolicyInformer, nspInformer kyvernov1informer.PolicyInformer) Cache {
pc := controller{
store: newPolicyCache(),
cpolLister: pInformer.Lister(),
polLister: nspInformer.Lister(),
pCounter: -1,
}
pInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: pc.addPolicy,
UpdateFunc: pc.updatePolicy,
DeleteFunc: pc.deletePolicy,
})
nspInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: pc.addNsPolicy,
UpdateFunc: pc.updateNsPolicy,
DeleteFunc: pc.deleteNsPolicy,
})
return &pc
}
func (c *controller) GetPolicies(pkey PolicyType, kind, nspace string) []kyvernov1.PolicyInterface {
var result []kyvernov1.PolicyInterface
result = append(result, c.store.get(pkey, kind, "")...)
result = append(result, c.store.get(pkey, "*", "")...)
if nspace != "" {
result = append(result, c.store.get(pkey, kind, nspace)...)
result = append(result, c.store.get(pkey, "*", nspace)...)
}
return result
}
func (c *controller) CheckPolicySync(stopCh <-chan struct{}) {
logger.Info("starting")
policies := []kyvernov1.PolicyInterface{}
polList, err := c.polLister.Policies(metav1.NamespaceAll).List(labels.Everything())
if err != nil {
logger.Error(err, "failed to list Policy")
os.Exit(1)
}
for _, p := range polList {
policies = append(policies, p)
}
cpolList, err := c.cpolLister.List(labels.Everything())
if err != nil {
logger.Error(err, "failed to list Cluster Policy")
os.Exit(1)
}
for _, p := range cpolList {
policies = append(policies, p)
}
atomic.StoreInt64(&c.pCounter, int64(len(policies)))
for _, policy := range policies {
c.store.set(policy)
atomic.AddInt64(&c.pCounter, ^int64(0))
}
if !c.hasPolicySynced() {
logger.Error(nil, "Failed to sync policy with cache")
os.Exit(1)
}
}
func (c *controller) addPolicy(obj interface{}) {
p := obj.(*kyvernov1.ClusterPolicy)
c.store.set(p)
}
func (c *controller) updatePolicy(old, cur interface{}) {
pOld := old.(*kyvernov1.ClusterPolicy)
pNew := cur.(*kyvernov1.ClusterPolicy)
if reflect.DeepEqual(pOld.Spec, pNew.Spec) {
return
}
c.store.set(pNew)
}
func (c *controller) deletePolicy(obj interface{}) {
p, ok := kubeutils.GetObjectWithTombstone(obj).(*kyvernov1.ClusterPolicy)
if ok {
c.store.unset(p)
} else {
logger.Info("Failed to get deleted object, the deleted cluster policy cannot be removed from the cache", "obj", obj)
}
}
func (c *controller) addNsPolicy(obj interface{}) {
p := obj.(*kyvernov1.Policy)
c.store.set(p)
}
func (c *controller) updateNsPolicy(old, cur interface{}) {
npOld := old.(*kyvernov1.Policy)
npNew := cur.(*kyvernov1.Policy)
if reflect.DeepEqual(npOld.Spec, npNew.Spec) {
return
}
c.store.set(npNew)
}
func (c *controller) deleteNsPolicy(obj interface{}) {
p, ok := kubeutils.GetObjectWithTombstone(obj).(*kyvernov1.Policy)
if ok {
c.store.unset(p)
} else {
logger.Info("Failed to get deleted object, the deleted policy cannot be removed from the cache", "obj", obj)
}
}
func (c *controller) hasPolicySynced() bool {
return atomic.LoadInt64(&c.pCounter) == 0
}

178
pkg/policycache/cache_test.go
View file

@ -2,53 +2,18 @@ package policycache
import (
"encoding/json"
"fmt"
"testing"
kyverno "github.com/kyverno/kyverno/api/kyverno/v1"
"github.com/kyverno/kyverno/pkg/autogen"
lv1 "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1"
"gotest.tools/assert"
"k8s.io/apimachinery/pkg/labels"
)
type dummyLister struct {
}
func (dl dummyLister) List(selector labels.Selector) (ret []*kyverno.ClusterPolicy, err error) {
return nil, fmt.Errorf("not implemented")
}
func (dl dummyLister) Get(name string) (*kyverno.ClusterPolicy, error) {
return nil, fmt.Errorf("not implemented")
}
func (dl dummyLister) ListResources(selector labels.Selector) (ret []*kyverno.ClusterPolicy, err error) {
return nil, fmt.Errorf("not implemented")
}
// type dymmyNsNamespace struct {}
type dummyNsLister struct {
}
func (dl dummyNsLister) Policies(name string) lv1.PolicyNamespaceLister {
return dummyNsLister{}
}
func (dl dummyNsLister) List(selector labels.Selector) (ret []*kyverno.Policy, err error) {
return nil, fmt.Errorf("not implemented")
}
func (dl dummyNsLister) Get(name string) (*kyverno.Policy, error) {
return nil, fmt.Errorf("not implemented")
}
func Test_All(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newPolicy(t)
//add
pCache.add(policy)
pCache.set(policy)
for _, rule := range autogen.ComputeRules(policy) {
for _, kind := range rule.MatchResources.Kinds {
@ -70,18 +35,18 @@ func Test_All(t *testing.T) {
}
// remove
pCache.remove(policy)
pCache.unset(policy)
kind := "pod"
validateEnforce := pCache.get(ValidateEnforce, kind, "")
assert.Assert(t, len(validateEnforce) == 0)
}
func Test_Add_Duplicate_Policy(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newPolicy(t)
pCache.add(policy)
pCache.add(policy)
pCache.add(policy)
pCache.set(policy)
pCache.set(policy)
pCache.set(policy)
for _, rule := range autogen.ComputeRules(policy) {
for _, kind := range rule.MatchResources.Kinds {
@ -103,35 +68,34 @@ func Test_Add_Duplicate_Policy(t *testing.T) {
}
func Test_Add_Validate_Audit(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newPolicy(t)
pCache.add(policy)
pCache.add(policy)
pCache.set(policy)
pCache.set(policy)
policy.Spec.ValidationFailureAction = "audit"
pCache.add(policy)
pCache.add(policy)
pCache.set(policy)
pCache.set(policy)
for _, rule := range autogen.ComputeRules(policy) {
for _, kind := range rule.MatchResources.Kinds {
validateEnforce := pCache.get(ValidateEnforce, kind, "")
if len(validateEnforce) != 1 {
t.Errorf("expected 1 validate policy, found %v", len(validateEnforce))
if len(validateEnforce) != 0 {
t.Errorf("expected 0 validate (enforce) policy, found %v", len(validateEnforce))
}
validateAudit := pCache.get(ValidateAudit, kind, "")
if len(validateEnforce) != 1 {
t.Errorf("expected 1 validate policy, found %v", len(validateAudit))
if len(validateAudit) != 1 {
t.Errorf("expected 1 validate (audit) policy, found %v", len(validateAudit))
}
}
}
}
func Test_Add_Remove(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newPolicy(t)
kind := "Pod"
pCache.add(policy)
pCache.set(policy)
validateEnforce := pCache.get(ValidateEnforce, kind, "")
if len(validateEnforce) != 1 {
@ -148,7 +112,7 @@ func Test_Add_Remove(t *testing.T) {
t.Errorf("expected 1 generate policy, found %v", len(generate))
}
pCache.remove(policy)
pCache.unset(policy)
deletedValidateEnforce := pCache.get(ValidateEnforce, kind, "")
if len(deletedValidateEnforce) != 0 {
t.Errorf("expected 0 validate enforce policy, found %v", len(deletedValidateEnforce))
@ -156,10 +120,10 @@ func Test_Add_Remove(t *testing.T) {
}
func Test_Add_Remove_Any(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newAnyPolicy(t)
kind := "Pod"
pCache.add(policy)
pCache.set(policy)
validateEnforce := pCache.get(ValidateEnforce, kind, "")
if len(validateEnforce) != 1 {
@ -176,7 +140,7 @@ func Test_Add_Remove_Any(t *testing.T) {
t.Errorf("expected 1 generate policy, found %v", len(generate))
}
pCache.remove(policy)
pCache.unset(policy)
deletedValidateEnforce := pCache.get(ValidateEnforce, kind, "")
if len(deletedValidateEnforce) != 0 {
t.Errorf("expected 0 validate enforce policy, found %v", len(deletedValidateEnforce))
@ -184,10 +148,10 @@ func Test_Add_Remove_Any(t *testing.T) {
}
func Test_Remove_From_Empty_Cache(t *testing.T) {
pCache := newPolicyCache(nil, nil)
pCache := newPolicyCache()
policy := newPolicy(t)
pCache.remove(policy)
pCache.unset(policy)
}
func newPolicy(t *testing.T) *kyverno.ClusterPolicy {
@ -678,7 +642,7 @@ func newUserTestPolicy(t *testing.T) kyverno.PolicyInterface {
return policy
}
func newgenratePolicy(t *testing.T) *kyverno.ClusterPolicy {
func newGeneratePolicy(t *testing.T) *kyverno.ClusterPolicy {
rawPolicy := []byte(`{
"metadata": {
"name": "add-networkpolicy",
@ -925,10 +889,10 @@ func newValidateEnforcePolicy(t *testing.T) *kyverno.ClusterPolicy {
}
func Test_Ns_All(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newNsPolicy(t)
//add
pCache.add(policy)
pCache.set(policy)
nspace := policy.GetNamespace()
for _, rule := range autogen.ComputeRules(policy) {
for _, kind := range rule.MatchResources.Kinds {
@ -950,18 +914,18 @@ func Test_Ns_All(t *testing.T) {
}
}
// remove
pCache.remove(policy)
pCache.unset(policy)
kind := "pod"
validateEnforce := pCache.get(ValidateEnforce, kind, nspace)
assert.Assert(t, len(validateEnforce) == 0)
}
func Test_Ns_Add_Duplicate_Policy(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newNsPolicy(t)
pCache.add(policy)
pCache.add(policy)
pCache.add(policy)
pCache.set(policy)
pCache.set(policy)
pCache.set(policy)
nspace := policy.GetNamespace()
for _, rule := range autogen.ComputeRules(policy) {
for _, kind := range rule.MatchResources.Kinds {
@ -984,42 +948,42 @@ func Test_Ns_Add_Duplicate_Policy(t *testing.T) {
}
func Test_Ns_Add_Validate_Audit(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newNsPolicy(t)
pCache.add(policy)
pCache.add(policy)
pCache.set(policy)
pCache.set(policy)
nspace := policy.GetNamespace()
policy.GetSpec().ValidationFailureAction = "audit"
pCache.add(policy)
pCache.add(policy)
pCache.set(policy)
pCache.set(policy)
for _, rule := range autogen.ComputeRules(policy) {
for _, kind := range rule.MatchResources.Kinds {
validateEnforce := pCache.get(ValidateEnforce, kind, nspace)
if len(validateEnforce) != 1 {
t.Errorf("expected 1 validate policy, found %v", len(validateEnforce))
if len(validateEnforce) != 0 {
t.Errorf("expected 0 validate (enforce) policy, found %v", len(validateEnforce))
}
validateAudit := pCache.get(ValidateAudit, kind, nspace)
if len(validateEnforce) != 1 {
t.Errorf("expected 1 validate policy, found %v", len(validateAudit))
if len(validateAudit) != 1 {
t.Errorf("expected 1 validate (audit) policy, found %v", len(validateAudit))
}
}
}
}
func Test_Ns_Add_Remove(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newNsPolicy(t)
nspace := policy.GetNamespace()
kind := "Pod"
pCache.add(policy)
pCache.set(policy)
validateEnforce := pCache.get(ValidateEnforce, kind, nspace)
if len(validateEnforce) != 1 {
t.Errorf("expected 1 validate enforce policy, found %v", len(validateEnforce))
}
pCache.remove(policy)
pCache.unset(policy)
deletedValidateEnforce := pCache.get(ValidateEnforce, kind, nspace)
if len(deletedValidateEnforce) != 0 {
t.Errorf("expected 0 validate enforce policy, found %v", len(deletedValidateEnforce))
@ -1027,10 +991,10 @@ func Test_Ns_Add_Remove(t *testing.T) {
}
func Test_GVk_Cache(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newGVKPolicy(t)
//add
pCache.add(policy)
pCache.set(policy)
for _, rule := range autogen.ComputeRules(policy) {
for _, kind := range rule.MatchResources.Kinds {
@ -1043,16 +1007,16 @@ func Test_GVk_Cache(t *testing.T) {
}
func Test_GVK_Add_Remove(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newGVKPolicy(t)
kind := "ClusterRole"
pCache.add(policy)
pCache.set(policy)
generate := pCache.get(Generate, kind, "")
if len(generate) != 1 {
t.Errorf("expected 1 generate policy, found %v", len(generate))
}
pCache.remove(policy)
pCache.unset(policy)
deletedGenerate := pCache.get(Generate, kind, "")
if len(deletedGenerate) != 0 {
t.Errorf("expected 0 generate policy, found %v", len(deletedGenerate))
@ -1060,11 +1024,11 @@ func Test_GVK_Add_Remove(t *testing.T) {
}
func Test_Add_Validate_Enforce(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newUserTestPolicy(t)
nspace := policy.GetNamespace()
//add
pCache.add(policy)
pCache.set(policy)
for _, rule := range autogen.ComputeRules(policy) {
for _, kind := range rule.MatchResources.Kinds {
validateEnforce := pCache.get(ValidateEnforce, kind, nspace)
@ -1076,17 +1040,17 @@ func Test_Add_Validate_Enforce(t *testing.T) {
}
func Test_Ns_Add_Remove_User(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newUserTestPolicy(t)
nspace := policy.GetNamespace()
kind := "Deployment"
pCache.add(policy)
pCache.set(policy)
validateEnforce := pCache.get(ValidateEnforce, kind, nspace)
if len(validateEnforce) != 1 {
t.Errorf("expected 1 validate enforce policy, found %v", len(validateEnforce))
}
pCache.remove(policy)
pCache.unset(policy)
deletedValidateEnforce := pCache.get(ValidateEnforce, kind, nspace)
if len(deletedValidateEnforce) != 0 {
t.Errorf("expected 0 validate enforce policy, found %v", len(deletedValidateEnforce))
@ -1094,12 +1058,12 @@ func Test_Ns_Add_Remove_User(t *testing.T) {
}
func Test_Mutate_Policy(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newMutatePolicy(t)
//add
pCache.add(policy)
pCache.add(policy)
pCache.add(policy)
pCache.set(policy)
pCache.set(policy)
pCache.set(policy)
for _, rule := range autogen.ComputeRules(policy) {
for _, kind := range rule.MatchResources.Kinds {
@ -1113,10 +1077,10 @@ func Test_Mutate_Policy(t *testing.T) {
}
func Test_Generate_Policy(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
policy := newgenratePolicy(t)
pCache := newPolicyCache()
policy := newGeneratePolicy(t)
//add
pCache.add(policy)
pCache.set(policy)
for _, rule := range autogen.ComputeRules(policy) {
for _, kind := range rule.MatchResources.Kinds {
@ -1130,14 +1094,14 @@ func Test_Generate_Policy(t *testing.T) {
}
func Test_NsMutate_Policy(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy := newMutatePolicy(t)
nspolicy := newNsMutatePolicy(t)
//add
pCache.add(policy)
pCache.add(nspolicy)
pCache.add(policy)
pCache.add(nspolicy)
pCache.set(policy)
pCache.set(nspolicy)
pCache.set(policy)
pCache.set(nspolicy)
nspace := policy.GetNamespace()
// get
@ -1155,11 +1119,11 @@ func Test_NsMutate_Policy(t *testing.T) {
}
func Test_Validate_Enforce_Policy(t *testing.T) {
pCache := newPolicyCache(dummyLister{}, dummyNsLister{})
pCache := newPolicyCache()
policy1 := newValidateAuditPolicy(t)
policy2 := newValidateEnforcePolicy(t)
pCache.add(policy1)
pCache.add(policy2)
pCache.set(policy1)
pCache.set(policy2)
validateEnforce := pCache.get(ValidateEnforce, "Pod", "")
if len(validateEnforce) != 2 {
@ -1171,8 +1135,8 @@ func Test_Validate_Enforce_Policy(t *testing.T) {
t.Errorf("adding: expected 0 validate audit policy, found %v", len(validateAudit))
}
pCache.remove(policy1)
pCache.remove(policy2)
pCache.unset(policy1)
pCache.unset(policy2)
validateEnforce = pCache.get(ValidateEnforce, "Pod", "")
if len(validateEnforce) != 0 {

142
pkg/policycache/informer.go
View file

@ -1,142 +0,0 @@
package policycache
import (
"os"
"reflect"
"sync/atomic"
kyverno "github.com/kyverno/kyverno/api/kyverno/v1"
kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1"
kyvernolister "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1"
kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/tools/cache"
)
// Controller is responsible for synchronizing Policy Cache,
// it embeds a policy informer to handle policy events.
// The cache is synced when a policy is add/update/delete.
// This cache is only used in the admission webhook to fast retrieve
// policies based on types (Mutate/ValidateEnforce/Generate/imageVerify).
type Controller struct {
Cache Interface
cpolLister kyvernolister.ClusterPolicyLister
polLister kyvernolister.PolicyLister
pCounter int64
}
// NewPolicyCacheController create a new PolicyController
func NewPolicyCacheController(pInformer kyvernoinformer.ClusterPolicyInformer, nspInformer kyvernoinformer.PolicyInformer) *Controller {
pc := Controller{
Cache: newPolicyCache(pInformer.Lister(), nspInformer.Lister()),
}
// ClusterPolicy Informer
pInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: pc.addPolicy,
UpdateFunc: pc.updatePolicy,
DeleteFunc: pc.deletePolicy,
})
// Policy Informer
nspInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: pc.addNsPolicy,
UpdateFunc: pc.updateNsPolicy,
DeleteFunc: pc.deleteNsPolicy,
})
pc.cpolLister = pInformer.Lister()
pc.polLister = nspInformer.Lister()
pc.pCounter = -1
return &pc
}
func (c *Controller) addPolicy(obj interface{}) {
p := obj.(*kyverno.ClusterPolicy)
c.Cache.add(p)
}
func (c *Controller) updatePolicy(old, cur interface{}) {
pOld := old.(*kyverno.ClusterPolicy)
pNew := cur.(*kyverno.ClusterPolicy)
if reflect.DeepEqual(pOld.Spec, pNew.Spec) {
return
}
c.Cache.update(pOld, pNew)
}
func (c *Controller) deletePolicy(obj interface{}) {
p, ok := kubeutils.GetObjectWithTombstone(obj).(*kyverno.ClusterPolicy)
if ok {
c.Cache.remove(p)
} else {
logger.Info("Failed to get deleted object, the deleted policy cannot be removed from the cache", "obj", obj)
}
}
// addNsPolicy - Add Policy to cache
func (c *Controller) addNsPolicy(obj interface{}) {
p := obj.(*kyverno.Policy)
c.Cache.add(p)
}
// updateNsPolicy - Update Policy of cache
func (c *Controller) updateNsPolicy(old, cur interface{}) {
npOld := old.(*kyverno.Policy)
npNew := cur.(*kyverno.Policy)
if reflect.DeepEqual(npOld.Spec, npNew.Spec) {
return
}
c.Cache.update(npOld, npNew)
}
// deleteNsPolicy - Delete Policy from cache
func (c *Controller) deleteNsPolicy(obj interface{}) {
p, ok := kubeutils.GetObjectWithTombstone(obj).(*kyverno.Policy)
if ok {
c.Cache.remove(p)
} else {
logger.Info("Failed to get deleted object, the deleted cluster policy cannot be removed from the cache", "obj", obj)
}
}
// CheckPolicySync wait until the internal policy cache is fully loaded
func (c *Controller) CheckPolicySync(stopCh <-chan struct{}) {
logger.Info("starting")
policies := []kyverno.PolicyInterface{}
polList, err := c.polLister.Policies(metav1.NamespaceAll).List(labels.Everything())
if err != nil {
logger.Error(err, "failed to list Policy")
os.Exit(1)
}
for _, p := range polList {
policies = append(policies, p)
}
cpolList, err := c.cpolLister.List(labels.Everything())
if err != nil {
logger.Error(err, "failed to list Cluster Policy")
os.Exit(1)
}
for _, p := range cpolList {
policies = append(policies, p)
}
atomic.StoreInt64(&c.pCounter, int64(len(policies)))
for _, policy := range policies {
c.Cache.add(policy)
atomic.AddInt64(&c.pCounter, ^int64(0))
}
if !c.hasPolicySynced() {
logger.Error(nil, "Failed to sync policy with cache")
os.Exit(1)
}
}
// hasPolicySynced check for policy counter zero
func (c *Controller) hasPolicySynced() bool {
return atomic.LoadInt64(&c.pCounter) == 0
}

112
pkg/policycache/policy_cache.go
View file

@ -1,112 +0,0 @@
package policycache
import (
kyverno "github.com/kyverno/kyverno/api/kyverno/v1"
kyvernolister "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1"
"github.com/kyverno/kyverno/pkg/policy"
kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
)
// Interface ...
// Interface get method use for to get policy names and mostly use to test cache testcases
type Interface interface {
// GetPolicies returns all policies that apply to a namespace, including cluster-wide policies
// If the namespace is empty, only cluster-wide policies are returned
GetPolicies(pType PolicyType, kind string, namespace string) []kyverno.PolicyInterface
// add adds a policy to the cache
add(kyverno.PolicyInterface)
// remove removes a policy from the cache
remove(kyverno.PolicyInterface)
// update update a policy from the cache
update(kyverno.PolicyInterface, kyverno.PolicyInterface)
get(PolicyType, string, string) []string
}
// policyCache ...
type policyCache struct {
pMap pMap
// list/get cluster policy resource
pLister kyvernolister.ClusterPolicyLister
// npLister can list/get namespace policy from the shared informer's store
npLister kyvernolister.PolicyLister
}
// newPolicyCache ...
func newPolicyCache(pLister kyvernolister.ClusterPolicyLister, npLister kyvernolister.PolicyLister) Interface {
namesCache := map[PolicyType]map[string]bool{
Mutate: make(map[string]bool),
ValidateEnforce: make(map[string]bool),
ValidateAudit: make(map[string]bool),
Generate: make(map[string]bool),
VerifyImagesMutate: make(map[string]bool),
VerifyImagesValidate: make(map[string]bool),
}
return &policyCache{
pMap{
nameCacheMap: namesCache,
kindDataMap: make(map[string]map[PolicyType][]string),
},
pLister,
npLister,
}
}
// Add a policy to cache
func (pc *policyCache) add(policy kyverno.PolicyInterface) {
pc.pMap.add(policy)
logger.V(4).Info("policy is added to cache", "name", policy.GetName())
}
// Get the list of matched policies
func (pc *policyCache) get(pkey PolicyType, kind, nspace string) []string {
return pc.pMap.get(pkey, kind, nspace)
}
func (pc *policyCache) GetPolicies(pkey PolicyType, kind, nspace string) []kyverno.PolicyInterface {
policies := pc.getPolicyObject(pkey, kind, "")
if nspace == "" {
return policies
}
nsPolicies := pc.getPolicyObject(pkey, kind, nspace)
return append(policies, nsPolicies...)
}
// Remove a policy from cache
func (pc *policyCache) remove(p kyverno.PolicyInterface) {
pc.pMap.remove(p)
logger.V(4).Info("policy is removed from cache", "name", p.GetName())
}
func (pc *policyCache) update(oldP kyverno.PolicyInterface, newP kyverno.PolicyInterface) {
pc.pMap.update(oldP, newP)
logger.V(4).Info("policy is updated from cache", "name", newP.GetName())
}
func (pc *policyCache) getPolicyObject(key PolicyType, gvk string, nspace string) (policyObject []kyverno.PolicyInterface) {
_, kind := kubeutils.GetKindFromGVK(gvk)
policyNames := pc.pMap.get(key, kind, nspace)
wildcardPolicies := pc.pMap.get(key, "*", nspace)
policyNames = append(policyNames, wildcardPolicies...)
for _, policyName := range policyNames {
var p kyverno.PolicyInterface
ns, key, isNamespacedPolicy := policy.ParseNamespacedPolicy(policyName)
if !isNamespacedPolicy {
p, _ = pc.pLister.Get(key)
} else {
if ns == nspace {
p, _ = pc.npLister.Policies(ns).Get(key)
}
}
if p != nil {
policyObject = append(policyObject, p)
}
}
return policyObject
}

202
pkg/policycache/policy_map.go
View file

@ -1,202 +0,0 @@
package policycache
import (
"sync"
kyverno "github.com/kyverno/kyverno/api/kyverno/v1"
"github.com/kyverno/kyverno/pkg/autogen"
"github.com/kyverno/kyverno/pkg/policy"
kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
)
type pMap struct {
lock sync.RWMutex
// kindDataMap field stores names of ClusterPolicies and Namespaced Policies.
// Since both the policy name use same type (i.e. string), Both policies can be differentiated based on
// "namespace". namespace policy get stored with policy namespace with policy name"
// kindDataMap {"kind": {{"policytype" : {"policyName","nsname/policyName}}},"kind2": {{"policytype" : {"nsname/policyName" }}}}
kindDataMap map[string]map[PolicyType][]string
// nameCacheMap stores the names of all existing policies in dataMap
// Policy names are stored as <namespace>/<name>
nameCacheMap map[PolicyType]map[string]bool
}
func (m *pMap) add(policy kyverno.PolicyInterface) {
m.lock.Lock()
defer m.lock.Unlock()
m.addPolicyToCache(policy)
}
func (m *pMap) addPolicyToCache(policy kyverno.PolicyInterface) {
spec := policy.GetSpec()
enforcePolicy := spec.GetValidationFailureAction() == kyverno.Enforce
for _, k := range spec.ValidationFailureActionOverrides {
if k.Action == kyverno.Enforce {
enforcePolicy = true
break
}
}
var pName = policy.GetName()
pSpace := policy.GetNamespace()
if pSpace != "" {
pName = pSpace + "/" + pName
}
for _, rule := range autogen.ComputeRules(policy) {
if len(rule.MatchResources.Any) > 0 {
for _, rmr := range rule.MatchResources.Any {
addCacheHelper(rmr, m, rule, pName, enforcePolicy)
}
} else if len(rule.MatchResources.All) > 0 {
for _, rmr := range rule.MatchResources.All {
addCacheHelper(rmr, m, rule, pName, enforcePolicy)
}
} else {
r := kyverno.ResourceFilter{UserInfo: rule.MatchResources.UserInfo, ResourceDescription: rule.MatchResources.ResourceDescription}
addCacheHelper(r, m, rule, pName, enforcePolicy)
}
}
}
func (m *pMap) get(key PolicyType, gvk, namespace string) (names []string) {
m.lock.RLock()
defer m.lock.RUnlock()
_, kind := kubeutils.GetKindFromGVK(gvk)
for _, policyName := range m.kindDataMap[kind][key] {
ns, key, isNamespacedPolicy := policy.ParseNamespacedPolicy(policyName)
if !isNamespacedPolicy && namespace == "" {
names = append(names, key)
} else {
if ns == namespace {
names = append(names, policyName)
}
}
}
return names
}
func (m *pMap) remove(policy kyverno.PolicyInterface) {
m.lock.Lock()
defer m.lock.Unlock()
m.removePolicyFromCache(policy)
}
func (m *pMap) removePolicyFromCache(policy kyverno.PolicyInterface) {
var pName = policy.GetName()
pSpace := policy.GetNamespace()
if pSpace != "" {
pName = pSpace + "/" + pName
}
for _, rule := range autogen.ComputeRules(policy) {
if len(rule.MatchResources.Any) > 0 {
for _, rmr := range rule.MatchResources.Any {
removeCacheHelper(rmr, m, pName)
}
} else if len(rule.MatchResources.All) > 0 {
for _, rmr := range rule.MatchResources.All {
removeCacheHelper(rmr, m, pName)
}
} else {
r := kyverno.ResourceFilter{UserInfo: rule.MatchResources.UserInfo, ResourceDescription: rule.MatchResources.ResourceDescription}
removeCacheHelper(r, m, pName)
}
}
}
func (m *pMap) update(old kyverno.PolicyInterface, new kyverno.PolicyInterface) {
m.lock.Lock()
defer m.lock.Unlock()
m.removePolicyFromCache(old)
m.addPolicyToCache(new)
}
func addCacheHelper(rmr kyverno.ResourceFilter, m *pMap, rule kyverno.Rule, pName string, enforcePolicy bool) {
for _, gvk := range rmr.Kinds {
_, k := kubeutils.GetKindFromGVK(gvk)
kind, _ := kubeutils.SplitSubresource(k)
_, ok := m.kindDataMap[kind]
if !ok {
m.kindDataMap[kind] = make(map[PolicyType][]string)
}
if rule.HasMutate() {
if !m.nameCacheMap[Mutate][kind+"/"+pName] {
m.nameCacheMap[Mutate][kind+"/"+pName] = true
mutatePolicy := m.kindDataMap[kind][Mutate]
m.kindDataMap[kind][Mutate] = append(mutatePolicy, pName)
}
continue
}
if rule.HasValidate() {
if enforcePolicy {
if !m.nameCacheMap[ValidateEnforce][kind+"/"+pName] {
m.nameCacheMap[ValidateEnforce][kind+"/"+pName] = true
validatePolicy := m.kindDataMap[kind][ValidateEnforce]
m.kindDataMap[kind][ValidateEnforce] = append(validatePolicy, pName)
}
continue
}
if !m.nameCacheMap[ValidateAudit][kind+"/"+pName] {
m.nameCacheMap[ValidateAudit][kind+"/"+pName] = true
validatePolicy := m.kindDataMap[kind][ValidateAudit]
m.kindDataMap[kind][ValidateAudit] = append(validatePolicy, pName)
}
continue
}
if rule.HasGenerate() {
if !m.nameCacheMap[Generate][kind+"/"+pName] {
m.nameCacheMap[Generate][kind+"/"+pName] = true
generatePolicy := m.kindDataMap[kind][Generate]
m.kindDataMap[kind][Generate] = append(generatePolicy, pName)
}
continue
}
if rule.HasVerifyImages() {
if !m.nameCacheMap[VerifyImagesMutate][kind+"/"+pName] {
m.nameCacheMap[VerifyImagesMutate][kind+"/"+pName] = true
imageVerifyMapPolicy := m.kindDataMap[kind][VerifyImagesMutate]
m.kindDataMap[kind][VerifyImagesMutate] = append(imageVerifyMapPolicy, pName)
}
if rule.HasImagesValidationChecks() {
m.nameCacheMap[VerifyImagesValidate][kind+"/"+pName] = true
imageVerifyMapPolicy := m.kindDataMap[kind][VerifyImagesValidate]
m.kindDataMap[kind][VerifyImagesValidate] = append(imageVerifyMapPolicy, pName)
}
continue
}
}
}
func removeCacheHelper(rmr kyverno.ResourceFilter, m *pMap, pName string) {
for _, gvk := range rmr.Kinds {
_, kind := kubeutils.GetKindFromGVK(gvk)
dataMap := m.kindDataMap[kind]
for policyType, policies := range dataMap {
var newPolicies []string
for _, p := range policies {
if p == pName {
continue
}
newPolicies = append(newPolicies, p)
}
m.kindDataMap[kind][policyType] = newPolicies
}
for _, nameCache := range m.nameCacheMap {
if ok := nameCache[kind+"/"+pName]; ok {
delete(nameCache, kind+"/"+pName)
}
}
}
}

172
pkg/policycache/store.go Normal file
View file

@ -0,0 +1,172 @@
package policycache
import (
"sync"
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
"github.com/kyverno/kyverno/pkg/autogen"
"github.com/kyverno/kyverno/pkg/policy"
kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
"k8s.io/apimachinery/pkg/util/sets"
)
type store interface {
// set inserts a policy in the cache
set(kyvernov1.PolicyInterface)
// unset removes a policy from the cache
unset(kyvernov1.PolicyInterface)
// get finds policies that match a given type, gvk and namespace
get(PolicyType, string, string) []kyvernov1.PolicyInterface
}
type policyCache struct {
store store
lock sync.RWMutex
}
func newPolicyCache() store {
return &policyCache{
store: newPolicyMap(),
}
}
func (pc *policyCache) set(policy kyvernov1.PolicyInterface) {
pc.lock.Lock()
defer pc.lock.Unlock()
pc.store.set(policy)
logger.V(4).Info("policy is added to cache", "name", policy.GetName())
}
func (pc *policyCache) unset(p kyvernov1.PolicyInterface) {
pc.lock.Lock()
defer pc.lock.Unlock()
pc.store.unset(p)
logger.V(4).Info("policy is removed from cache", "name", p.GetName())
}
func (pc *policyCache) get(pkey PolicyType, kind, nspace string) []kyvernov1.PolicyInterface {
pc.lock.RLock()
defer pc.lock.RUnlock()
return pc.store.get(pkey, kind, nspace)
}
type policyMap struct {
// policies maps names to policy interfaces
policies map[string]kyvernov1.PolicyInterface
// kindType stores names of ClusterPolicies and Namespaced Policies.
// Since both the policy name use same type (i.e. string), Both policies can be differentiated based on
// "namespace". namespace policy get stored with policy namespace with policy name"
// kindDataMap {"kind": {{"policytype" : {"policyName","nsname/policyName}}},"kind2": {{"policytype" : {"nsname/policyName" }}}}
kindType map[string]map[PolicyType]sets.String
}
func newPolicyMap() *policyMap {
return &policyMap{
policies: map[string]kyvernov1.PolicyInterface{},
kindType: map[string]map[PolicyType]sets.String{},
}
}
func computeKind(gvk string) string {
_, k := kubeutils.GetKindFromGVK(gvk)
kind, _ := kubeutils.SplitSubresource(k)
return kind
}
func computeKey(policy kyvernov1.PolicyInterface) string {
namespace := policy.GetNamespace()
if namespace == "" {
return policy.GetName()
} else {
return namespace + "/" + policy.GetName()
}
}
func computeEnforcePolicy(spec *kyvernov1.Spec) bool {
if spec.GetValidationFailureAction() == kyvernov1.Enforce {
return true
}
for _, k := range spec.ValidationFailureActionOverrides {
if k.Action == kyvernov1.Enforce {
return true
}
}
return false
}
func set(set sets.String, item string, value bool) sets.String {
if value {
return set.Insert(item)
} else {
return set.Delete(item)
}
}
func (m *policyMap) set(policy kyvernov1.PolicyInterface) {
key, enforcePolicy := computeKey(policy), computeEnforcePolicy(policy.GetSpec())
m.policies[key] = policy
type state struct {
hasMutate, hasValidate, hasGenerate, hasVerifyImages, hasImagesValidationChecks bool
}
kindStates := map[string]state{}
for _, rule := range autogen.ComputeRules(policy) {
for _, gvk := range rule.MatchResources.GetKinds() {
kind := computeKind(gvk)
entry := kindStates[kind]
entry.hasMutate = (entry.hasMutate || rule.HasMutate())
entry.hasValidate = (entry.hasValidate || rule.HasValidate())
entry.hasGenerate = (entry.hasGenerate || rule.HasGenerate())
entry.hasVerifyImages = (entry.hasVerifyImages || rule.HasVerifyImages())
entry.hasImagesValidationChecks = (entry.hasImagesValidationChecks || rule.HasImagesValidationChecks())
kindStates[kind] = entry
}
}
for kind, state := range kindStates {
if m.kindType[kind] == nil {
m.kindType[kind] = map[PolicyType]sets.String{
Mutate: sets.NewString(),
ValidateEnforce: sets.NewString(),
ValidateAudit: sets.NewString(),
Generate: sets.NewString(),
VerifyImagesMutate: sets.NewString(),
VerifyImagesValidate: sets.NewString(),
}
}
m.kindType[kind][Mutate] = set(m.kindType[kind][Mutate], key, state.hasMutate)
m.kindType[kind][ValidateEnforce] = set(m.kindType[kind][ValidateEnforce], key, state.hasValidate && enforcePolicy)
m.kindType[kind][ValidateAudit] = set(m.kindType[kind][ValidateAudit], key, state.hasValidate && !enforcePolicy)
m.kindType[kind][Generate] = set(m.kindType[kind][Generate], key, state.hasGenerate)
m.kindType[kind][VerifyImagesMutate] = set(m.kindType[kind][VerifyImagesMutate], key, state.hasVerifyImages)
m.kindType[kind][VerifyImagesValidate] = set(m.kindType[kind][VerifyImagesValidate], key, state.hasVerifyImages && state.hasImagesValidationChecks)
}
}
func (m *policyMap) unset(policy kyvernov1.PolicyInterface) {
key := computeKey(policy)
delete(m.policies, key)
for kind := range m.kindType {
for policyType := range m.kindType[kind] {
m.kindType[kind][policyType] = m.kindType[kind][policyType].Delete(key)
}
}
}
func (m *policyMap) get(key PolicyType, gvk, namespace string) []kyvernov1.PolicyInterface {
kind := computeKind(gvk)
var result []kyvernov1.PolicyInterface
for policyName := range m.kindType[kind][key] {
ns, _, isNamespacedPolicy := policy.ParseNamespacedPolicy(policyName)
policy := m.policies[policyName]
if policy == nil {
logger.Info("nil policy in the cache, this should not happen")
}
if !isNamespacedPolicy && namespace == "" {
result = append(result, policy)
} else {
if ns == namespace {
result = append(result, policy)
}
}
}
return result
}

4
pkg/webhooks/server.go
View file

@ -65,7 +65,7 @@ type WebhookServer struct {
eventGen event.Interface
// policy cache
pCache policycache.Interface
pCache policycache.Cache
// webhook registration client
webhookRegister *webhookconfig.Register
@ -113,7 +113,7 @@ func NewWebhookServer(
crInformer rbacinformer.ClusterRoleInformer,
namespace informers.NamespaceInformer,
eventGen event.Interface,
pCache policycache.Interface,
pCache policycache.Cache,
webhookRegistrationClient *webhookconfig.Register,
webhookMonitor *webhookconfig.Monitor,
configHandler config.Configuration,

4
pkg/webhooks/validate_audit.go
View file

@ -44,7 +44,7 @@ type AuditHandler interface {
type auditHandler struct {
client client.Interface
queue workqueue.RateLimitingInterface
pCache policycache.Interface
pCache policycache.Cache
eventGen event.Interface
prGenerator policyreport.GeneratorInterface
@ -58,7 +58,7 @@ type auditHandler struct {
}
// NewValidateAuditHandler returns a new instance of audit policy handler
func NewValidateAuditHandler(pCache policycache.Interface,
func NewValidateAuditHandler(pCache policycache.Cache,
eventGen event.Interface,
prGenerator policyreport.GeneratorInterface,
rbInformer rbacinformer.RoleBindingInformer,