feat: reuse --protectManagedResources flag in the cleanup controller (#8566)

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>
Co-authored-by: shuting <shuting@nirmata.com>

charts/kyverno/templates/cleanup-controller/deployment.yaml

@@ -121,6 +121,7 @@ spec:
               "dumpPayload"
               "logging"
               "ttlController"
+              "protectManagedResources"
             ) | nindent 12 }}
             {{- range $key, $value := .Values.cleanupController.extraArgs }}
             {{- if $value }}

cmd/cleanup-controller/main.go

@@ -25,6 +25,7 @@ import (
 	"github.com/kyverno/kyverno/pkg/leaderelection"
 	"github.com/kyverno/kyverno/pkg/logging"
 	"github.com/kyverno/kyverno/pkg/tls"
+	"github.com/kyverno/kyverno/pkg/toggle"
 	"github.com/kyverno/kyverno/pkg/webhooks"
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -76,6 +77,7 @@ func main() {
 	flagset.IntVar(&webhookServerPort, "webhookServerPort", 9443, "Port used by the webhook server.")
 	flagset.IntVar(&maxQueuedEvents, "maxQueuedEvents", 1000, "Maximum events to be queued.")
 	flagset.DurationVar(&interval, "ttlReconciliationInterval", time.Minute, "Set this flag to set the interval after which the resource controller reconciliation should occur")
+	flagset.Func(toggle.ProtectManagedResourcesFlagName, toggle.ProtectManagedResourcesDescription, toggle.ProtectManagedResources.Parse)
 	flagset.StringVar(&caSecretName, "caSecretName", "", "Name of the secret containing CA.")
 	flagset.StringVar(&tlsSecretName, "tlsSecretName", "", "Name of the secret containing TLS pair.")
 	flagset.DurationVar(&renewBefore, "renewBefore", 15*24*time.Hour, "The certificate renewal time before expiration")

config/install-latest-testing.yaml

@@ -51762,6 +51762,7 @@ spec:
             - --dumpPayload=false
             - --loggingFormat=text
             - --v=2
+            - --protectManagedResources=false
             - --ttlReconciliationInterval=1m
             
           env:

pkg/controllers/cleanup/controller.go

@@ -22,6 +22,7 @@ import (
 	"github.com/kyverno/kyverno/pkg/event"
 	"github.com/kyverno/kyverno/pkg/logging"
 	"github.com/kyverno/kyverno/pkg/metrics"
+	"github.com/kyverno/kyverno/pkg/toggle"
 	"github.com/kyverno/kyverno/pkg/utils/conditions"
 	controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
 	"github.com/kyverno/kyverno/pkg/utils/match"
@@ -216,7 +217,11 @@ func (c *controller) cleanup(ctx context.Context, logger logr.Logger, policy kyv
 				namespace := resource.GetNamespace()
 				name := resource.GetName()
 				debug := debug.WithValues("name", name, "namespace", namespace)
-				if !controllerutils.IsManagedByKyverno(&resource) {
+				// check if the resource is owned by Kyverno
+				if controllerutils.IsManagedByKyverno(&resource) && toggle.FromContext(ctx).ProtectManagedResources() {
+					continue
+				}
+
 				var nsLabels map[string]string
 				if namespace != "" {
 					ns, err := c.nsLister.Get(namespace)
@@ -264,6 +269,32 @@ func (c *controller) cleanup(ctx context.Context, logger logr.Logger, policy kyv
 					}
 				}
 				// check conditions
+				if spec.Conditions != nil {
+					enginectx.Reset()
+					if err := enginectx.SetTargetResource(resource.Object); err != nil {
+						debug.Error(err, "failed to add resource in context")
+						errs = append(errs, err)
+						continue
+					}
+					if spec.ExcludeResources != nil {
+						excluded := match.CheckMatchesResources(
+							resource,
+							*spec.ExcludeResources,
+							nsLabels,
+							// TODO(eddycharly): we don't have user info here, we should check that
+							// we don't have user conditions in the policy rule
+							kyvernov1beta1.RequestInfo{},
+							resource.GroupVersionKind(),
+							"",
+						)
+						if excluded == nil {
+							debug.Info("resource/exclude matched")
+							continue
+						} else {
+							debug.Info("resource/exclude didn't match", "result", excluded)
+						}
+					}
+					// check conditions
 					if spec.Conditions != nil {
 						enginectx.Reset()
 						if err := enginectx.SetTargetResource(resource.Object); err != nil {
@@ -296,6 +327,34 @@ func (c *controller) cleanup(ctx context.Context, logger logr.Logger, policy kyv
 					labels = append(labels, commonLabels...)
 					labels = append(labels, attribute.String("resource_namespace", namespace))
 					logger.WithValues("name", name, "namespace", namespace).Info("resource matched, it will be deleted...")
+					if err := c.client.DeleteResource(ctx, resource.GetAPIVersion(), resource.GetKind(), namespace, name, false); err != nil {
+						if c.metrics.cleanupFailuresTotal != nil {
+							c.metrics.cleanupFailuresTotal.Add(ctx, 1, metric.WithAttributes(labels...))
+						}
+						debug.Error(err, "failed to delete resource")
+						errs = append(errs, err)
+						continue
+					}
+					if err := enginectx.AddImageInfos(&resource, c.configuration); err != nil {
+						debug.Error(err, "failed to add image infos in context")
+						errs = append(errs, err)
+						continue
+					}
+					passed, err := conditions.CheckAnyAllConditions(logger, enginectx, *spec.Conditions)
+					if err != nil {
+						debug.Error(err, "failed to check condition")
+						errs = append(errs, err)
+						continue
+					}
+					if !passed {
+						debug.Info("conditions did not pass")
+						continue
+					}
+				}
+				var labels []attribute.KeyValue
+				labels = append(labels, commonLabels...)
+				labels = append(labels, attribute.String("resource_namespace", namespace))
+				logger.WithValues("name", name, "namespace", namespace).Info("resource matched, it will be deleted...")
 				if err := c.client.DeleteResource(ctx, resource.GetAPIVersion(), resource.GetKind(), namespace, name, false); err != nil {
 					if c.metrics.cleanupFailuresTotal != nil {
 						c.metrics.cleanupFailuresTotal.Add(ctx, 1, metric.WithAttributes(labels...))
@@ -315,7 +374,6 @@ func (c *controller) cleanup(ctx context.Context, logger logr.Logger, policy kyv
 			}
 		}
 	}
-	}
 	return multierr.Combine(errs...)
 }