From 2eb9660aeed9ec3ef698fe43de3992f6d2f9c268 Mon Sep 17 00:00:00 2001 From: shuting Date: Sat, 22 Jan 2022 13:36:42 +0800 Subject: [PATCH] Reduce throttling requests for Kyverno resources (#3042) * remove resoureCache from the event controller Signed-off-by: ShutingZhao * create rcr using typed client to reduce PUT throttling request Signed-off-by: ShutingZhao * use typed client for report/rcr operations Signed-off-by: ShutingZhao * clarify naming patterns for Kyverno ClusterRoles/ClusterRoleBindings (#3029) * clarify naming patterns for Kyverno ClusterRoles/ClusterRoleBindings (#3032) * fix comment * fix comment Co-authored-by: Vyankatesh Kudtarkar --- pkg/policyreport/changerequestcreator.go | 37 ----------- pkg/policyreport/conversion.go | 78 ++++++++++++++++++++++++ pkg/policyreport/reportcontroller.go | 52 +++++++++++++--- 3 files changed, 120 insertions(+), 47 deletions(-) create mode 100644 pkg/policyreport/conversion.go diff --git a/pkg/policyreport/changerequestcreator.go b/pkg/policyreport/changerequestcreator.go index 2ee2349261..3d419481bc 100644 --- a/pkg/policyreport/changerequestcreator.go +++ b/pkg/policyreport/changerequestcreator.go @@ -3,20 +3,17 @@ package policyreport import ( "context" "crypto/rand" - "encoding/json" "math/big" "reflect" "sync" "time" "github.com/go-logr/logr" - report "github.com/kyverno/kyverno/api/kyverno/v1alpha2" policyreportclient "github.com/kyverno/kyverno/pkg/client/clientset/versioned" "github.com/kyverno/kyverno/pkg/config" "github.com/patrickmn/go-cache" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" ) // creator is an interface that buffers report change requests @@ -281,37 +278,3 @@ func isDeleteRequest(request *unstructured.Unstructured) bool { return false } - -func convertToRCR(request *unstructured.Unstructured) (*report.ReportChangeRequest, error) { - rcr := report.ReportChangeRequest{} - raw, err := request.MarshalJSON() - if err != nil { - return nil, err - } - - err = json.Unmarshal(raw, &rcr) - rcr.SetGroupVersionKind(schema.GroupVersionKind{ - Group: report.SchemeGroupVersion.Group, - Version: report.SchemeGroupVersion.Version, - Kind: "ReportChangeRequest", - }) - - return &rcr, err -} - -func convertToCRCR(request *unstructured.Unstructured) (*report.ClusterReportChangeRequest, error) { - rcr := report.ClusterReportChangeRequest{} - raw, err := request.MarshalJSON() - if err != nil { - return nil, err - } - - err = json.Unmarshal(raw, &rcr) - rcr.SetGroupVersionKind(schema.GroupVersionKind{ - Group: report.SchemeGroupVersion.Group, - Version: report.SchemeGroupVersion.Version, - Kind: "ClusterReportChangeRequest", - }) - - return &rcr, err -} diff --git a/pkg/policyreport/conversion.go b/pkg/policyreport/conversion.go new file mode 100644 index 0000000000..6c927fd8f6 --- /dev/null +++ b/pkg/policyreport/conversion.go @@ -0,0 +1,78 @@ +package policyreport + +import ( + "encoding/json" + + typercr "github.com/kyverno/kyverno/api/kyverno/v1alpha2" + report "github.com/kyverno/kyverno/api/policyreport/v1alpha2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func convertToRCR(request *unstructured.Unstructured) (*typercr.ReportChangeRequest, error) { + rcr := typercr.ReportChangeRequest{} + raw, err := request.MarshalJSON() + if err != nil { + return nil, err + } + + err = json.Unmarshal(raw, &rcr) + rcr.SetGroupVersionKind(schema.GroupVersionKind{ + Group: typercr.SchemeGroupVersion.Group, + Version: typercr.SchemeGroupVersion.Version, + Kind: "ReportChangeRequest", + }) + + return &rcr, err +} + +func convertToCRCR(request *unstructured.Unstructured) (*typercr.ClusterReportChangeRequest, error) { + rcr := typercr.ClusterReportChangeRequest{} + raw, err := request.MarshalJSON() + if err != nil { + return nil, err + } + + err = json.Unmarshal(raw, &rcr) + rcr.SetGroupVersionKind(schema.GroupVersionKind{ + Group: typercr.SchemeGroupVersion.Group, + Version: typercr.SchemeGroupVersion.Version, + Kind: "ClusterReportChangeRequest", + }) + + return &rcr, err +} + +func convertToPolr(request *unstructured.Unstructured) (*report.PolicyReport, error) { + polr := report.PolicyReport{} + raw, err := request.MarshalJSON() + if err != nil { + return nil, err + } + + err = json.Unmarshal(raw, &polr) + polr.SetGroupVersionKind(schema.GroupVersionKind{ + Group: report.SchemeGroupVersion.Group, + Version: report.SchemeGroupVersion.Version, + Kind: "PolicyReport", + }) + + return &polr, err +} + +func convertToCpolr(request *unstructured.Unstructured) (*report.ClusterPolicyReport, error) { + cpolr := report.ClusterPolicyReport{} + raw, err := request.MarshalJSON() + if err != nil { + return nil, err + } + + err = json.Unmarshal(raw, &cpolr) + cpolr.SetGroupVersionKind(schema.GroupVersionKind{ + Group: report.SchemeGroupVersion.Group, + Version: report.SchemeGroupVersion.Version, + Kind: "ClusterPolicyReport", + }) + + return &cpolr, err +} diff --git a/pkg/policyreport/reportcontroller.go b/pkg/policyreport/reportcontroller.go index 76cf83492c..401041b9e5 100644 --- a/pkg/policyreport/reportcontroller.go +++ b/pkg/policyreport/reportcontroller.go @@ -1,6 +1,7 @@ package policyreport import ( + "context" "fmt" "reflect" "strings" @@ -382,7 +383,12 @@ func (g *ReportGenerator) createReportIfNotPresent(namespace string, new *unstru report, err = g.reportLister.PolicyReports(namespace).Get(GeneratePolicyReportName(namespace)) if err != nil { if apierrors.IsNotFound(err) && new != nil { - if _, err := g.dclient.CreateResource(new.GetAPIVersion(), new.GetKind(), new.GetNamespace(), new, false); err != nil { + polr, err := convertToPolr(new) + if err != nil { + return nil, fmt.Errorf("failed to convert to policyReport: %v", err) + } + + if _, err := g.pclient.Wgpolicyk8sV1alpha2().PolicyReports(new.GetNamespace()).Create(context.TODO(), polr, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("failed to create policyReport: %v", err) } @@ -398,7 +404,12 @@ func (g *ReportGenerator) createReportIfNotPresent(namespace string, new *unstru if err != nil { if apierrors.IsNotFound(err) { if new != nil { - if _, err := g.dclient.CreateResource(new.GetAPIVersion(), new.GetKind(), new.GetNamespace(), new, false); err != nil { + cpolr, err := convertToCpolr(new) + if err != nil { + return nil, fmt.Errorf("failed to convert to ClusterPolicyReport: %v", err) + } + + if _, err := g.pclient.Wgpolicyk8sV1alpha2().ClusterPolicyReports().Create(context.TODO(), cpolr, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("failed to create ClusterPolicyReport: %v", err) } @@ -459,7 +470,7 @@ func (g *ReportGenerator) removeFromClusterPolicyReport(policyName, ruleName str cpolr.Summary = calculateSummary(newRes) gv := report.SchemeGroupVersion cpolr.SetGroupVersionKind(schema.GroupVersionKind{Group: gv.Group, Version: gv.Version, Kind: "ClusterPolicyReport"}) - if _, err := g.dclient.UpdateResource("", "ClusterPolicyReport", "", cpolr, false); err != nil { + if _, err := g.pclient.Wgpolicyk8sV1alpha2().ClusterPolicyReports().Update(context.TODO(), cpolr, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("failed to update clusterPolicyReport %s %v", cpolr.Name, err) } } @@ -473,6 +484,10 @@ func (g *ReportGenerator) removeFromPolicyReport(policyName, ruleName string) er } selector, err := metav1.LabelSelectorAsSelector(LabelSelector) + if err != nil { + g.log.Error(err, "failed to build labelSelector") + } + policyReports := []*report.PolicyReport{} for _, ns := range namespaces.Items { reports, err := g.reportLister.PolicyReports(ns.GetName()).List(selector) @@ -498,7 +513,7 @@ func (g *ReportGenerator) removeFromPolicyReport(policyName, ruleName string) er gv := report.SchemeGroupVersion gvk := schema.GroupVersionKind{Group: gv.Group, Version: gv.Version, Kind: "PolicyReport"} r.SetGroupVersionKind(gvk) - if _, err := g.dclient.UpdateResource("", "PolicyReport", r.GetNamespace(), r, false); err != nil { + if _, err := g.pclient.Wgpolicyk8sV1alpha2().PolicyReports(r.GetNamespace()).Update(context.TODO(), r, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("failed to update PolicyReport %s %v", r.GetName(), err) } } @@ -647,7 +662,7 @@ func (g *ReportGenerator) updateReport(old interface{}, new *unstructured.Unstru if oldTyped, ok := old.(*report.ClusterPolicyReport); ok { if oldTyped.GetDeletionTimestamp() != nil { - return g.dclient.DeleteResource(oldTyped.APIVersion, "ClusterPolicyReport", oldTyped.Namespace, oldTyped.Name, false) + return g.pclient.Wgpolicyk8sV1alpha2().ClusterPolicyReports().Delete(context.TODO(), oldTyped.Name, metav1.DeleteOptions{}) } if oldUnstructured, err = runtime.DefaultUnstructuredConverter.ToUnstructured(oldTyped); err != nil { @@ -657,7 +672,7 @@ func (g *ReportGenerator) updateReport(old interface{}, new *unstructured.Unstru new.SetResourceVersion(oldTyped.GetResourceVersion()) } else if oldTyped, ok := old.(*report.PolicyReport); ok { if oldTyped.GetDeletionTimestamp() != nil { - return g.dclient.DeleteResource(oldTyped.APIVersion, "PolicyReport", oldTyped.Namespace, oldTyped.Name, false) + return g.pclient.Wgpolicyk8sV1alpha2().PolicyReports(oldTyped.Namespace).Delete(context.TODO(), oldTyped.Name, metav1.DeleteOptions{}) } if oldUnstructured, err = runtime.DefaultUnstructuredConverter.ToUnstructured(oldTyped); err != nil { @@ -680,8 +695,25 @@ func (g *ReportGenerator) updateReport(old interface{}, new *unstructured.Unstru return nil } - if _, err = g.dclient.UpdateResource(new.GetAPIVersion(), new.GetKind(), new.GetNamespace(), new, false); err != nil { - return fmt.Errorf("failed to update policy report: %v", err) + if new.GetKind() == "PolicyReport" { + polr, err := convertToPolr(new) + if err != nil { + return fmt.Errorf("error converting to PolicyReport: %v", err) + } + if _, err := g.pclient.Wgpolicyk8sV1alpha2().PolicyReports(new.GetNamespace()).Update(context.TODO(), polr, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update PolicyReport: %v", err) + } + } + + if new.GetKind() == "ClusterPolicyReport" { + cpolr, err := convertToCpolr(new) + if err != nil { + return fmt.Errorf("error converting to ClusterPolicyReport: %v", err) + } + if _, err := g.pclient.Wgpolicyk8sV1alpha2().ClusterPolicyReports().Update(context.TODO(), cpolr, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update ClusterPolicyReport: %v", err) + } + } g.log.V(3).Info("successfully updated policy report", "kind", new.GetKind(), "namespace", new.GetNamespace(), "name", new.GetName()) @@ -692,7 +724,7 @@ func (g *ReportGenerator) cleanupReportRequests(requestsGeneral interface{}) { defer g.log.V(5).Info("successfully cleaned up report requests") if requests, ok := requestsGeneral.([]*changerequest.ReportChangeRequest); ok { for _, request := range requests { - if err := g.dclient.DeleteResource(request.APIVersion, "ReportChangeRequest", config.KyvernoNamespace, request.Name, false); err != nil { + if err := g.pclient.KyvernoV1alpha2().ReportChangeRequests(config.KyvernoNamespace).Delete(context.TODO(), request.Name, metav1.DeleteOptions{}); err != nil { if !apierrors.IsNotFound(err) { g.log.Error(err, "failed to delete report request") } @@ -702,7 +734,7 @@ func (g *ReportGenerator) cleanupReportRequests(requestsGeneral interface{}) { if requests, ok := requestsGeneral.([]*changerequest.ClusterReportChangeRequest); ok { for _, request := range requests { - if err := g.dclient.DeleteResource(request.APIVersion, "ClusterReportChangeRequest", "", request.Name, false); err != nil { + if err := g.pclient.KyvernoV1alpha2().ClusterReportChangeRequests().Delete(context.TODO(), request.Name, metav1.DeleteOptions{}); err != nil { if !apierrors.IsNotFound(err) { g.log.Error(err, "failed to delete clusterReportChangeRequest") }