From bdb675b9c0bbc58fadf4d637848d0775e06a6869 Mon Sep 17 00:00:00 2001 From: Prateek Pandey Date: Tue, 29 Mar 2022 18:34:33 +0530 Subject: [PATCH] feat: generate support for namespace policy (#3472) * feat: generate support for namespace policy Signed-off-by: prateekpandey14 * use policy spec instead Signed-off-by: prateekpandey14 * refactor the changes Signed-off-by: prateekpandey14 * add synced flag for Namespace policies Signed-off-by: prateekpandey14 --- cmd/kyverno/main.go | 2 ++ pkg/engine/generation.go | 49 +++++++++++++++++++++++++++++ pkg/generate/cleanup/controller.go | 46 ++++++++++++++++++++++----- pkg/generate/generate.go | 35 ++++++++++++++++++--- pkg/generate/generate_controller.go | 12 ++++++- pkg/webhooks/generate/generate.go | 8 +++-- pkg/webhooks/generation.go | 9 +++++- 7 files changed, 145 insertions(+), 16 deletions(-) diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index bb494e8c10..56bf9d6524 100755 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -337,6 +337,7 @@ func main() { pclient, client, pInformer.Kyverno().V1().ClusterPolicies(), + pInformer.Kyverno().V1().Policies(), pInformer.Kyverno().V1().GenerateRequests(), eventGenerator, kubedynamicInformer, @@ -355,6 +356,7 @@ func main() { pclient, client, pInformer.Kyverno().V1().ClusterPolicies(), + pInformer.Kyverno().V1().Policies(), pInformer.Kyverno().V1().GenerateRequests(), kubedynamicInformer, log.Log.WithName("GenerateCleanUpController"), diff --git a/pkg/engine/generation.go b/pkg/engine/generation.go index feab929b80..5c6bf72b3a 100644 --- a/pkg/engine/generation.go +++ b/pkg/engine/generation.go @@ -5,6 +5,7 @@ import ( "github.com/kyverno/kyverno/pkg/autogen" "github.com/kyverno/kyverno/pkg/engine/common" + "k8s.io/client-go/tools/cache" kyverno "github.com/kyverno/kyverno/api/kyverno/v1" "github.com/kyverno/kyverno/pkg/engine/response" @@ -21,6 +22,54 @@ func Generate(policyContext *PolicyContext) (resp *response.EngineResponse) { return filterRules(policyContext, policyStartTime) } +// GenerateResponse checks for validity of generate rule on the resource +func GenerateResponse(policyContext *PolicyContext, gr kyverno.GenerateRequest) (resp *response.EngineResponse) { + policyStartTime := time.Now() + return filterGenerateRules(policyContext, gr.Spec.Policy, policyStartTime) +} + +func filterGenerateRules(policyContext *PolicyContext, policyNameKey string, startTime time.Time) *response.EngineResponse { + kind := policyContext.NewResource.GetKind() + name := policyContext.NewResource.GetName() + namespace := policyContext.NewResource.GetNamespace() + apiVersion := policyContext.NewResource.GetAPIVersion() + pNamespace, pName, err := cache.SplitMetaNamespaceKey(policyNameKey) + if err != nil { + log.Log.Error(err, "failed to spilt name and namespace", policyNameKey) + } + + resp := &response.EngineResponse{ + PolicyResponse: response.PolicyResponse{ + Policy: response.PolicySpec{ + Name: pName, + Namespace: pNamespace, + }, + PolicyStats: response.PolicyStats{ + PolicyExecutionTimestamp: startTime.Unix(), + }, + Resource: response.ResourceSpec{ + Kind: kind, + Name: name, + Namespace: namespace, + APIVersion: apiVersion, + }, + }, + } + + if policyContext.ExcludeResourceFunc(kind, namespace, name) { + log.Log.WithName("Generate").Info("resource excluded", "kind", kind, "namespace", namespace, "name", name) + return resp + } + + for _, rule := range autogen.ComputeRules(&policyContext.Policy) { + if ruleResp := filterRule(rule, policyContext); ruleResp != nil { + resp.PolicyResponse.Rules = append(resp.PolicyResponse.Rules, *ruleResp) + } + } + + return resp +} + func filterRules(policyContext *PolicyContext, startTime time.Time) *response.EngineResponse { kind := policyContext.NewResource.GetKind() name := policyContext.NewResource.GetName() diff --git a/pkg/generate/cleanup/controller.go b/pkg/generate/cleanup/controller.go index d4e7395851..9b959d0f59 100644 --- a/pkg/generate/cleanup/controller.go +++ b/pkg/generate/cleanup/controller.go @@ -50,12 +50,18 @@ type Controller struct { // pLister can list/get cluster policy from the shared informer's store pLister kyvernolister.ClusterPolicyLister + // npLister can list/get namespace policy from the shared informer's store + npLister kyvernolister.PolicyLister + // grLister can list/get generate request from the shared informer's store grLister kyvernolister.GenerateRequestNamespaceLister // pSynced returns true if the cluster policy has been synced at least once pSynced cache.InformerSynced + // pSynced returns true if the Namespace policy has been synced at least once + npSynced cache.InformerSynced + // grSynced returns true if the generate request store has been synced at least once grSynced cache.InformerSynced @@ -75,6 +81,7 @@ func NewController( kyvernoclient *kyvernoclient.Clientset, client *dclient.Client, pInformer kyvernoinformer.ClusterPolicyInformer, + npInformer kyvernoinformer.PolicyInformer, grInformer kyvernoinformer.GenerateRequestInformer, dynamicInformer dynamicinformer.DynamicSharedInformerFactory, log logr.Logger, @@ -92,9 +99,11 @@ func NewController( c.control = Control{client: kyvernoclient} c.pLister = pInformer.Lister() + c.npLister = npInformer.Lister() c.grLister = grInformer.Lister().GenerateRequests(config.KyvernoNamespace) c.pSynced = pInformer.Informer().HasSynced + c.npSynced = npInformer.Informer().HasSynced c.grSynced = grInformer.Informer().HasSynced gvr, err := client.DiscoveryClient.GetGVRFromKind("Namespace") @@ -249,7 +258,7 @@ func (c *Controller) Run(workers int, stopCh <-chan struct{}) { logger.Info("starting") defer logger.Info("shutting down") - if !cache.WaitForCacheSync(stopCh, c.pSynced, c.grSynced) { + if !cache.WaitForCacheSync(stopCh, c.pSynced, c.grSynced, c.npSynced) { logger.Info("failed to sync informer cache") return } @@ -338,16 +347,37 @@ func (c *Controller) syncGenerateRequest(key string) error { return err } - _, err = c.pLister.Get(gr.Spec.Policy) + pNamespace, pName, err := cache.SplitMetaNamespaceKey(gr.Spec.Policy) if err != nil { - if !apierrors.IsNotFound(err) { - return err - } - err = c.control.Delete(gr.Name) + return err + } + + if pNamespace == "" { + _, err = c.pLister.Get(pName) if err != nil { - return err + if !apierrors.IsNotFound(err) { + return err + } + logger.Error(err, "failed to get clusterpolicy, deleting the generate request") + err = c.control.Delete(gr.Name) + if err != nil { + return err + } + return nil + } + } else { + _, err = c.npLister.Policies(pNamespace).Get(pName) + if err != nil { + if !apierrors.IsNotFound(err) { + return err + } + logger.Error(err, "failed to get policy, deleting the generate request") + err = c.control.Delete(gr.Name) + if err != nil { + return err + } + return nil } - return nil } return c.processGR(*gr) } diff --git a/pkg/generate/generate.go b/pkg/generate/generate.go index 8359da5d07..3b8bd84a20 100644 --- a/pkg/generate/generate.go +++ b/pkg/generate/generate.go @@ -28,6 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" ) func (c *Controller) processGR(gr *kyverno.GenerateRequest) error { @@ -121,7 +122,7 @@ func (c *Controller) applyGenerate(resource unstructured.Unstructured, gr kyvern logger.V(3).Info("applying generate policy rule") - policyObj, err := c.policyLister.Get(gr.Spec.Policy) + policySpec, err := c.getPolicySpec(gr) if err != nil { if apierrors.IsNotFound(err) { for _, e := range gr.Status.GeneratedResources { @@ -137,7 +138,6 @@ func (c *Controller) applyGenerate(resource unstructured.Unstructured, gr kyvern } } } - return nil, false, nil } @@ -189,9 +189,12 @@ func (c *Controller) applyGenerate(resource unstructured.Unstructured, gr kyvern logger.Error(err, "unable to add image info to variables context") } + policy := kyverno.ClusterPolicy{ + Spec: policySpec, + } policyContext := &engine.PolicyContext{ NewResource: resource, - Policy: *policyObj, + Policy: policy, AdmissionInfo: gr.Spec.Context.UserRequestInfo, ExcludeGroupRole: c.Config.GetExcludeGroupRole(), ExcludeResourceFunc: c.Config.ToFilter, @@ -201,7 +204,7 @@ func (c *Controller) applyGenerate(resource unstructured.Unstructured, gr kyvern } // check if the policy still applies to the resource - engineResponse := engine.Generate(policyContext) + engineResponse := engine.GenerateResponse(policyContext, gr) if len(engineResponse.PolicyResponse.Rules) == 0 { logger.V(4).Info(doesNotApply) return nil, false, errors.New(doesNotApply) @@ -239,6 +242,30 @@ func (c *Controller) applyGenerate(resource unstructured.Unstructured, gr kyvern return c.applyGeneratePolicy(logger, policyContext, gr, applicableRules) } +// getPolicySpec gets the policy spec from the ClusterPolicy/Policy +func (c *Controller) getPolicySpec(gr kyverno.GenerateRequest) (kyverno.Spec, error) { + var policySpec kyverno.Spec + + pNamespace, pName, err := cache.SplitMetaNamespaceKey(gr.Spec.Policy) + if err != nil { + return policySpec, err + } + + if pNamespace == "" { + policyObj, err := c.policyLister.Get(pName) + if err != nil { + return policySpec, err + } + return policyObj.Spec, err + } else { + npolicyObj, err := c.npolicyLister.Policies(pNamespace).Get(pName) + if err != nil { + return policySpec, err + } + return npolicyObj.Spec, nil + } +} + func updateStatus(statusControl StatusControlInterface, gr kyverno.GenerateRequest, err error, genResources []kyverno.ResourceSpec, precreatedResource bool) error { if err != nil { return statusControl.Failed(gr, err.Error(), genResources) diff --git a/pkg/generate/generate_controller.go b/pkg/generate/generate_controller.go index b4953a1f64..4ee3d85e1d 100644 --- a/pkg/generate/generate_controller.go +++ b/pkg/generate/generate_controller.go @@ -52,12 +52,18 @@ type Controller struct { // policyLister can list/get cluster policy from the shared informer's store policyLister kyvernolister.ClusterPolicyLister + // policyLister can list/get Namespace policy from the shared informer's store + npolicyLister kyvernolister.PolicyLister + // grLister can list/get generate request from the shared informer's store grLister kyvernolister.GenerateRequestNamespaceLister // policySynced returns true if the Cluster policy store has been synced at least once policySynced cache.InformerSynced + // policySynced returns true if the Namespace policy store has been synced at least once + npolicySynced cache.InformerSynced + // grSynced returns true if the Generate Request store has been synced at least once grSynced cache.InformerSynced @@ -77,6 +83,7 @@ func NewController( kyvernoClient *kyvernoclient.Clientset, client *dclient.Client, policyInformer kyvernoinformer.ClusterPolicyInformer, + npolicyInformer kyvernoinformer.PolicyInformer, grInformer kyvernoinformer.GenerateRequestInformer, eventGen event.Interface, dynamicInformer dynamicinformer.DynamicSharedInformerFactory, @@ -99,6 +106,8 @@ func NewController( c.policySynced = policyInformer.Informer().HasSynced + c.npolicySynced = npolicyInformer.Informer().HasSynced + c.grSynced = grInformer.Informer().HasSynced grInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: c.addGR, @@ -107,6 +116,7 @@ func NewController( }) c.policyLister = policyInformer.Lister() + c.npolicyLister = npolicyInformer.Lister() c.grLister = grInformer.Lister().GenerateRequests(config.KyvernoNamespace) gvr, err := client.DiscoveryClient.GetGVRFromKind("Namespace") @@ -125,7 +135,7 @@ func (c *Controller) Run(workers int, stopCh <-chan struct{}) { defer c.queue.ShutDown() defer c.log.Info("shutting down") - if !cache.WaitForCacheSync(stopCh, c.policySynced, c.grSynced) { + if !cache.WaitForCacheSync(stopCh, c.policySynced, c.grSynced, c.npolicySynced) { c.log.Info("failed to sync informer cache") return } diff --git a/pkg/webhooks/generate/generate.go b/pkg/webhooks/generate/generate.go index fe5a72089c..1f94efefcf 100644 --- a/pkg/webhooks/generate/generate.go +++ b/pkg/webhooks/generate/generate.go @@ -109,6 +109,10 @@ func retryApplyResource(client *kyvernoclient.Clientset, grSpec kyverno.Generate var i int var err error + _, policyName, err := cache.SplitMetaNamespaceKey(grSpec.Policy) + if err != nil { + return err + } applyResource := func() error { gr := kyverno.GenerateRequest{ Spec: grSpec, @@ -121,7 +125,7 @@ func retryApplyResource(client *kyvernoclient.Clientset, grSpec kyverno.Generate if action == v1beta1.Create || action == v1beta1.Update { log.V(4).Info("querying all generate requests") selector := labels.SelectorFromSet(labels.Set(map[string]string{ - "generate.kyverno.io/policy-name": grSpec.Policy, + "generate.kyverno.io/policy-name": policyName, "generate.kyverno.io/resource-name": grSpec.Resource.Name, "generate.kyverno.io/resource-kind": grSpec.Resource.Kind, "generate.kyverno.io/resource-namespace": grSpec.Resource.Namespace, @@ -154,7 +158,7 @@ func retryApplyResource(client *kyvernoclient.Clientset, grSpec kyverno.Generate if !isExist { gr.SetGenerateName("gr-") gr.SetLabels(map[string]string{ - "generate.kyverno.io/policy-name": grSpec.Policy, + "generate.kyverno.io/policy-name": policyName, "generate.kyverno.io/resource-name": grSpec.Resource.Name, "generate.kyverno.io/resource-kind": grSpec.Resource.Kind, "generate.kyverno.io/resource-namespace": grSpec.Resource.Namespace, diff --git a/pkg/webhooks/generation.go b/pkg/webhooks/generation.go index 8a94e29bd8..a63c79f360 100644 --- a/pkg/webhooks/generation.go +++ b/pkg/webhooks/generation.go @@ -443,8 +443,15 @@ func applyGenerateRequest(request *v1beta1.AdmissionRequest, gnGenerator generat } func transform(admissionRequestInfo kyverno.AdmissionRequestInfoObject, userRequestInfo kyverno.RequestInfo, er *response.EngineResponse) kyverno.GenerateRequestSpec { + var PolicyNameNamespaceKey string + if er.PolicyResponse.Policy.Namespace != "" { + PolicyNameNamespaceKey = fmt.Sprintf("%s", er.PolicyResponse.Policy.Namespace+"/"+er.PolicyResponse.Policy.Name) + } else { + PolicyNameNamespaceKey = er.PolicyResponse.Policy.Name + } + gr := kyverno.GenerateRequestSpec{ - Policy: er.PolicyResponse.Policy.Name, + Policy: PolicyNameNamespaceKey, Resource: kyverno.ResourceSpec{ Kind: er.PolicyResponse.Resource.Kind, Namespace: er.PolicyResponse.Resource.Namespace,