feat: reuse --protectManagedResources flag in the cleanup controller (#8566)

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>
Co-authored-by: shuting <shuting@nirmata.com>

charts/kyverno/templates/cleanup-controller/deployment.yaml

@@ -121,6 +121,7 @@ spec:
               "dumpPayload"
               "logging"
               "ttlController"
+              "protectManagedResources"
             ) | nindent 12 }}
             {{- range $key, $value := .Values.cleanupController.extraArgs }}
             {{- if $value }}

cmd/cleanup-controller/main.go

@@ -25,6 +25,7 @@ import (
 	"github.com/kyverno/kyverno/pkg/leaderelection"
 	"github.com/kyverno/kyverno/pkg/logging"
 	"github.com/kyverno/kyverno/pkg/tls"
+	"github.com/kyverno/kyverno/pkg/toggle"
 	"github.com/kyverno/kyverno/pkg/webhooks"
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -76,6 +77,7 @@ func main() {
 	flagset.IntVar(&webhookServerPort, "webhookServerPort", 9443, "Port used by the webhook server.")
 	flagset.IntVar(&maxQueuedEvents, "maxQueuedEvents", 1000, "Maximum events to be queued.")
 	flagset.DurationVar(&interval, "ttlReconciliationInterval", time.Minute, "Set this flag to set the interval after which the resource controller reconciliation should occur")
+	flagset.Func(toggle.ProtectManagedResourcesFlagName, toggle.ProtectManagedResourcesDescription, toggle.ProtectManagedResources.Parse)
 	flagset.StringVar(&caSecretName, "caSecretName", "", "Name of the secret containing CA.")
 	flagset.StringVar(&tlsSecretName, "tlsSecretName", "", "Name of the secret containing TLS pair.")
 	flagset.DurationVar(&renewBefore, "renewBefore", 15*24*time.Hour, "The certificate renewal time before expiration")

config/install-latest-testing.yaml

@@ -51762,6 +51762,7 @@ spec:
             - --dumpPayload=false
             - --loggingFormat=text
             - --v=2
+            - --protectManagedResources=false
             - --ttlReconciliationInterval=1m
             
           env:

pkg/controllers/cleanup/controller.go

@@ -22,6 +22,7 @@ import (
 	"github.com/kyverno/kyverno/pkg/event"
 	"github.com/kyverno/kyverno/pkg/logging"
 	"github.com/kyverno/kyverno/pkg/metrics"
+	"github.com/kyverno/kyverno/pkg/toggle"
 	"github.com/kyverno/kyverno/pkg/utils/conditions"
 	controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
 	"github.com/kyverno/kyverno/pkg/utils/match"
@@ -216,24 +217,43 @@ func (c *controller) cleanup(ctx context.Context, logger logr.Logger, policy kyv
 				namespace := resource.GetNamespace()
 				name := resource.GetName()
 				debug := debug.WithValues("name", name, "namespace", namespace)
-				if !controllerutils.IsManagedByKyverno(&resource) {
+				// check if the resource is owned by Kyverno
-					var nsLabels map[string]string
+				if controllerutils.IsManagedByKyverno(&resource) && toggle.FromContext(ctx).ProtectManagedResources() {
-					if namespace != "" {
+					continue
-						ns, err := c.nsLister.Get(namespace)
+				}
-						if err != nil {
+
-							debug.Error(err, "failed to get namespace labels")
+				var nsLabels map[string]string
-							errs = append(errs, err)
+				if namespace != "" {
-						}
+					ns, err := c.nsLister.Get(namespace)
-						nsLabels = ns.GetLabels()
+					if err != nil {
+						debug.Error(err, "failed to get namespace labels")
+						errs = append(errs, err)
 					}
-					// match namespaces
+					nsLabels = ns.GetLabels()
-					if err := match.CheckNamespace(policy.GetNamespace(), resource); err != nil {
+				}
-						debug.Info("resource namespace didn't match policy namespace", "result", err)
+				// match namespaces
-					}
+				if err := match.CheckNamespace(policy.GetNamespace(), resource); err != nil {
-					// match resource with match/exclude clause
+					debug.Info("resource namespace didn't match policy namespace", "result", err)
-					matched := match.CheckMatchesResources(
+				}
+				// match resource with match/exclude clause
+				matched := match.CheckMatchesResources(
+					resource,
+					spec.MatchResources,
+					nsLabels,
+					// TODO(eddycharly): we don't have user info here, we should check that
+					// we don't have user conditions in the policy rule
+					kyvernov1beta1.RequestInfo{},
+					resource.GroupVersionKind(),
+					"",
+				)
+				if matched != nil {
+					debug.Info("resource/match didn't match", "result", matched)
+					continue
+				}
+				if spec.ExcludeResources != nil {
+					excluded := match.CheckMatchesResources(
 						resource,
-						spec.MatchResources,
+						*spec.ExcludeResources,
 						nsLabels,
 						// TODO(eddycharly): we don't have user info here, we should check that
 						// we don't have user conditions in the policy rule
@@ -241,8 +261,19 @@ func (c *controller) cleanup(ctx context.Context, logger logr.Logger, policy kyv
 						resource.GroupVersionKind(),
 						"",
 					)
-					if matched != nil {
+					if excluded == nil {
-						debug.Info("resource/match didn't match", "result", matched)
+						debug.Info("resource/exclude matched")
+						continue
+					} else {
+						debug.Info("resource/exclude didn't match", "result", excluded)
+					}
+				}
+				// check conditions
+				if spec.Conditions != nil {
+					enginectx.Reset()
+					if err := enginectx.SetTargetResource(resource.Object); err != nil {
+						debug.Error(err, "failed to add resource in context")
+						errs = append(errs, err)
 						continue
 					}
 					if spec.ExcludeResources != nil {
@@ -302,16 +333,43 @@ func (c *controller) cleanup(ctx context.Context, logger logr.Logger, policy kyv
 						}
 						debug.Error(err, "failed to delete resource")
 						errs = append(errs, err)
-						e := event.NewCleanupPolicyEvent(policy, resource, err)
+						continue
-						c.eventGen.Add(e)
-					} else {
-						if c.metrics.deletedObjectsTotal != nil {
-							c.metrics.deletedObjectsTotal.Add(ctx, 1, metric.WithAttributes(labels...))
-						}
-						debug.Info("deleted")
-						e := event.NewCleanupPolicyEvent(policy, resource, nil)
-						c.eventGen.Add(e)
 					}
+					if err := enginectx.AddImageInfos(&resource, c.configuration); err != nil {
+						debug.Error(err, "failed to add image infos in context")
+						errs = append(errs, err)
+						continue
+					}
+					passed, err := conditions.CheckAnyAllConditions(logger, enginectx, *spec.Conditions)
+					if err != nil {
+						debug.Error(err, "failed to check condition")
+						errs = append(errs, err)
+						continue
+					}
+					if !passed {
+						debug.Info("conditions did not pass")
+						continue
+					}
+				}
+				var labels []attribute.KeyValue
+				labels = append(labels, commonLabels...)
+				labels = append(labels, attribute.String("resource_namespace", namespace))
+				logger.WithValues("name", name, "namespace", namespace).Info("resource matched, it will be deleted...")
+				if err := c.client.DeleteResource(ctx, resource.GetAPIVersion(), resource.GetKind(), namespace, name, false); err != nil {
+					if c.metrics.cleanupFailuresTotal != nil {
+						c.metrics.cleanupFailuresTotal.Add(ctx, 1, metric.WithAttributes(labels...))
+					}
+					debug.Error(err, "failed to delete resource")
+					errs = append(errs, err)
+					e := event.NewCleanupPolicyEvent(policy, resource, err)
+					c.eventGen.Add(e)
+				} else {
+					if c.metrics.deletedObjectsTotal != nil {
+						c.metrics.deletedObjectsTotal.Add(ctx, 1, metric.WithAttributes(labels...))
+					}
+					debug.Info("deleted")
+					e := event.NewCleanupPolicyEvent(policy, resource, nil)
+					c.eventGen.Add(e)
 				}
 			}
 		}