Cleanup Report Change Requests (#2134)

* clean up RCRs if retry fails

Signed-off-by: Shuting Zhao <shutting06@gmail.com>

* cleanup report change request when background scan starts

Signed-off-by: Shuting Zhao <shutting06@gmail.com>

* add verb deletecollection to ClusterRole kyverno:customresources

Signed-off-by: Shuting Zhao <shutting06@gmail.com>

charts/kyverno/templates/clusterrole.yaml

@@ -118,6 +118,7 @@ rules:
   - patch
   - update
   - watch
+  - deletecollection
 - apiGroups:
   - 'apiextensions.k8s.io'
   resources:

definitions/k8s-resource/clusterroles.yaml

@@ -118,6 +118,7 @@ rules:
   - patch
   - update
   - watch
+  - deletecollection
 - apiGroups:
   - 'apiextensions.k8s.io'
   resources:

pkg/policy/report.go

@@ -9,7 +9,9 @@ import (
 	"github.com/go-logr/logr"
 	v1alpha1 "github.com/kyverno/kyverno/pkg/api/policyreport/v1alpha1"
 	kyvernoclient "github.com/kyverno/kyverno/pkg/client/clientset/versioned"
+	changerequestlister "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1alpha1"
 	policyreportlister "github.com/kyverno/kyverno/pkg/client/listers/policyreport/v1alpha1"
+	"github.com/kyverno/kyverno/pkg/config"
 	"github.com/kyverno/kyverno/pkg/engine/response"
 	"github.com/kyverno/kyverno/pkg/event"
 	"github.com/kyverno/kyverno/pkg/policyreport"
@@ -44,6 +46,10 @@ func (pc *PolicyController) forceReconciliation(reconcileCh <-chan bool, stopCh
 		select {
 		case <-ticker.C:
 			logger.Info("performing the background scan", "scan interval", pc.reconcilePeriod.String())
+			if err := pc.policyReportEraser.CleanupReportChangeRequests(cleanupReportChangeRequests); err != nil {
+				logger.Error(err, "failed to cleanup report change requests")
+			}
+
 			if err := pc.policyReportEraser.EraseResultsEntries(eraseResultsEntries); err != nil {
 				logger.Error(err, "continue reconciling policy reports")
 			}
@@ -52,6 +58,10 @@ func (pc *PolicyController) forceReconciliation(reconcileCh <-chan bool, stopCh
 
 		case erase := <-reconcileCh:
 			logger.Info("received the reconcile signal, reconciling policy report")
+			if err := pc.policyReportEraser.CleanupReportChangeRequests(cleanupReportChangeRequests); err != nil {
+				logger.Error(err, "failed to cleanup report change requests")
+			}
+
 			if erase {
 				if err := pc.policyReportEraser.EraseResultsEntries(eraseResultsEntries); err != nil {
 					logger.Error(err, "continue reconciling policy reports")
@@ -66,6 +76,29 @@ func (pc *PolicyController) forceReconciliation(reconcileCh <-chan bool, stopCh
 	}
 }
 
+func cleanupReportChangeRequests(pclient *kyvernoclient.Clientset, rcrLister changerequestlister.ReportChangeRequestLister, crcrLister changerequestlister.ClusterReportChangeRequestLister) error {
+	var errors []string
+
+	var gracePeriod int64 = 0
+	deleteOptions := metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod}
+
+	err := pclient.KyvernoV1alpha1().ClusterReportChangeRequests().DeleteCollection(context.TODO(), deleteOptions, metav1.ListOptions{})
+	if err != nil {
+		errors = append(errors, err.Error())
+	}
+
+	err = pclient.KyvernoV1alpha1().ReportChangeRequests(config.KyvernoNamespace).DeleteCollection(context.TODO(), deleteOptions, metav1.ListOptions{})
+	if err != nil {
+		errors = append(errors, err.Error())
+	}
+
+	if len(errors) == 0 {
+		return nil
+	}
+
+	return fmt.Errorf("%v", strings.Join(errors, ";"))
+}
+
 func eraseResultsEntries(pclient *kyvernoclient.Clientset, reportLister policyreportlister.PolicyReportLister, clusterReportLister policyreportlister.ClusterPolicyReportLister) error {
 	var errors []string
 

pkg/policyreport/policyreport.go

@@ -9,16 +9,23 @@ import (
 	changerequest "github.com/kyverno/kyverno/pkg/api/kyverno/v1alpha1"
 	report "github.com/kyverno/kyverno/pkg/api/policyreport/v1alpha1"
 	kyvernoclient "github.com/kyverno/kyverno/pkg/client/clientset/versioned"
+	changerequestlister "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1alpha1"
 	policyreportlister "github.com/kyverno/kyverno/pkg/client/listers/policyreport/v1alpha1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
 
 type PolicyReportEraser interface {
+	CleanupReportChangeRequests(cleanup CleanupReportChangeRequests) error
 	EraseResultsEntries(erase EraseResultsEntries) error
 }
 
+type CleanupReportChangeRequests = func(pclient *kyvernoclient.Clientset, rcrLister changerequestlister.ReportChangeRequestLister, crcrLister changerequestlister.ClusterReportChangeRequestLister) error
 type EraseResultsEntries = func(pclient *kyvernoclient.Clientset, reportLister policyreportlister.PolicyReportLister, clusterReportLister policyreportlister.ClusterPolicyReportLister) error
 
+func (g *ReportGenerator) CleanupReportChangeRequests(cleanup CleanupReportChangeRequests) error {
+	return cleanup(g.pclient, g.reportChangeRequestLister, g.clusterReportChangeRequestLister)
+}
+
 func (g *ReportGenerator) EraseResultsEntries(erase EraseResultsEntries) error {
 	return erase(g.pclient, g.reportLister, g.clusterReportLister)
 }

pkg/policyreport/reportcontroller.go

@@ -247,13 +247,13 @@ func (g *ReportGenerator) processNextWorkItem() bool {
 		return true
 	}
 
-	err := g.syncHandler(keyStr)
-	g.handleErr(err, key)
+	aggregatedRequests, err := g.syncHandler(keyStr)
+	g.handleErr(err, key, aggregatedRequests)
 
 	return true
 }
 
-func (g *ReportGenerator) handleErr(err error, key interface{}) {
+func (g *ReportGenerator) handleErr(err error, key interface{}, aggregatedRequests interface{}) {
 	logger := g.log
 	if err == nil {
 		g.queue.Forget(key)
@@ -269,11 +269,15 @@ func (g *ReportGenerator) handleErr(err error, key interface{}) {
 
 	logger.Error(err, "failed to process policy report", "key", key)
 	g.queue.Forget(key)
+
+	if aggregatedRequests != nil {
+		g.cleanupReportRequests(aggregatedRequests)
+	}
 }
 
 // syncHandler reconciles clusterPolicyReport if namespace == ""
 // otherwise it updates policyReport
-func (g *ReportGenerator) syncHandler(key string) error {
+func (g *ReportGenerator) syncHandler(key string) (aggregatedRequests interface{}, err error) {
 	g.log.V(4).Info("syncing policy report", "key", key)
 
 	if policy, rule, ok := isDeletedPolicyKey(key); ok {
@@ -283,24 +287,24 @@ func (g *ReportGenerator) syncHandler(key string) error {
 	namespace := key
 	new, aggregatedRequests, err := g.aggregateReports(namespace)
 	if err != nil {
-		return fmt.Errorf("failed to aggregate reportChangeRequest results %v", err)
+		return aggregatedRequests, fmt.Errorf("failed to aggregate reportChangeRequest results %v", err)
 	}
 
 	var old interface{}
 	if old, err = g.createReportIfNotPresent(namespace, new, aggregatedRequests); err != nil {
-		return err
+		return aggregatedRequests, err
 	}
 	if old == nil {
 		g.cleanupReportRequests(aggregatedRequests)
-		return nil
+		return nil, nil
 	}
 
 	if err := g.updateReport(old, new, aggregatedRequests); err != nil {
-		return err
+		return aggregatedRequests, err
 	}
 
 	g.cleanupReportRequests(aggregatedRequests)
-	return nil
+	return nil, nil
 }
 
 // createReportIfNotPresent creates cluster / policyReport if not present
@@ -362,13 +366,13 @@ func (g *ReportGenerator) createReportIfNotPresent(namespace string, new *unstru
 	return report, nil
 }
 
-func (g *ReportGenerator) removePolicyEntryFromReport(policyName, ruleName string) error {
+func (g *ReportGenerator) removePolicyEntryFromReport(policyName, ruleName string) (aggregatedRequests interface{}, err error) {
 	if err := g.removeFromClusterPolicyReport(policyName, ruleName); err != nil {
-		return err
+		return nil, err
 	}
 
 	if err := g.removeFromPolicyReport(policyName, ruleName); err != nil {
-		return err
+		return nil, err
 	}
 
 	labelset := labels.Set(map[string]string{deletedLabelPolicy: policyName})
@@ -378,13 +382,13 @@ func (g *ReportGenerator) removePolicyEntryFromReport(policyName, ruleName strin
 			deletedLabelRule:   ruleName,
 		})
 	}
-	aggregatedRequests, err := g.reportChangeRequestLister.ReportChangeRequests(config.KyvernoNamespace).List(labels.SelectorFromSet(labelset))
+	aggregatedRequests, err = g.reportChangeRequestLister.ReportChangeRequests(config.KyvernoNamespace).List(labels.SelectorFromSet(labelset))
 	if err != nil {
-		return err
+		return aggregatedRequests, err
 	}
 
 	g.cleanupReportRequests(aggregatedRequests)
-	return nil
+	return nil, nil
 }
 
 func (g *ReportGenerator) removeFromClusterPolicyReport(policyName, ruleName string) error {