package cleanup import ( "context" "time" "github.com/go-logr/logr" kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" kyvernov1beta1 "github.com/kyverno/kyverno/api/kyverno/v1beta1" kyvernov2alpha1 "github.com/kyverno/kyverno/api/kyverno/v2alpha1" kyvernov2alpha1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v2alpha1" "github.com/kyverno/kyverno/pkg/clients/dclient" "github.com/kyverno/kyverno/pkg/config" engineapi "github.com/kyverno/kyverno/pkg/engine/api" enginecontext "github.com/kyverno/kyverno/pkg/engine/context" "github.com/kyverno/kyverno/pkg/engine/factories" "github.com/kyverno/kyverno/pkg/engine/jmespath" "github.com/kyverno/kyverno/pkg/event" "github.com/kyverno/kyverno/pkg/metrics" controllerutils "github.com/kyverno/kyverno/pkg/utils/controller" "github.com/kyverno/kyverno/pkg/utils/match" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" "go.uber.org/multierr" "k8s.io/apimachinery/pkg/util/sets" corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" ) type handlers struct { client dclient.Interface cpolLister kyvernov2alpha1listers.ClusterCleanupPolicyLister polLister kyvernov2alpha1listers.CleanupPolicyLister nsLister corev1listers.NamespaceLister cmResolver engineapi.ConfigmapResolver eventGen event.Interface jp jmespath.Interface metrics cleanupMetrics } type cleanupMetrics struct { deletedObjectsTotal metric.Int64Counter cleanupFailuresTotal metric.Int64Counter } func newCleanupMetrics(logger logr.Logger) cleanupMetrics { meter := otel.GetMeterProvider().Meter(metrics.MeterName) deletedObjectsTotal, err := meter.Int64Counter( "kyverno_cleanup_controller_deletedobjects", metric.WithDescription("can be used to track number of deleted objects."), ) if err != nil { logger.Error(err, "Failed to create instrument, cleanup_controller_deletedobjects_total") } cleanupFailuresTotal, err := meter.Int64Counter( "kyverno_cleanup_controller_errors", metric.WithDescription("can be used to track number of cleanup failures."), ) if err != nil { logger.Error(err, "Failed to create instrument, cleanup_controller_errors_total") } return cleanupMetrics{ deletedObjectsTotal: deletedObjectsTotal, cleanupFailuresTotal: cleanupFailuresTotal, } } func New( logger logr.Logger, client dclient.Interface, cpolLister kyvernov2alpha1listers.ClusterCleanupPolicyLister, polLister kyvernov2alpha1listers.CleanupPolicyLister, nsLister corev1listers.NamespaceLister, cmResolver engineapi.ConfigmapResolver, jp jmespath.Interface, eventGen event.Interface, ) *handlers { return &handlers{ client: client, cpolLister: cpolLister, polLister: polLister, nsLister: nsLister, cmResolver: cmResolver, eventGen: eventGen, metrics: newCleanupMetrics(logger), jp: jp, } } func (h *handlers) Cleanup(ctx context.Context, logger logr.Logger, name string, _ time.Time, cfg config.Configuration) error { logger.Info("cleaning up...") defer logger.Info("done") namespace, name, err := cache.SplitMetaNamespaceKey(name) if err != nil { return err } policy, err := h.lookupPolicy(namespace, name) if err != nil { return err } return h.executePolicy(ctx, logger, policy, cfg) } func (h *handlers) lookupPolicy(namespace, name string) (kyvernov2alpha1.CleanupPolicyInterface, error) { if namespace == "" { return h.cpolLister.Get(name) } else { return h.polLister.CleanupPolicies(namespace).Get(name) } } func (h *handlers) executePolicy( ctx context.Context, logger logr.Logger, policy kyvernov2alpha1.CleanupPolicyInterface, cfg config.Configuration, ) error { spec := policy.GetSpec() kinds := sets.New(spec.MatchResources.GetKinds()...) debug := logger.V(4) var errs []error enginectx := enginecontext.NewContext(h.jp) ctxFactory := factories.DefaultContextLoaderFactory(h.cmResolver) loader := ctxFactory(nil, kyvernov1.Rule{}) if err := loader.Load( ctx, h.jp, h.client, nil, spec.Context, enginectx, ); err != nil { return err } for kind := range kinds { commonLabels := []attribute.KeyValue{ attribute.String("policy_type", policy.GetKind()), attribute.String("policy_namespace", policy.GetNamespace()), attribute.String("policy_name", policy.GetName()), attribute.String("resource_kind", kind), } debug := debug.WithValues("kind", kind) debug.Info("processing...") list, err := h.client.ListResource(ctx, "", kind, policy.GetNamespace(), nil) if err != nil { debug.Error(err, "failed to list resources") errs = append(errs, err) if h.metrics.cleanupFailuresTotal != nil { h.metrics.cleanupFailuresTotal.Add(ctx, 1, metric.WithAttributes(commonLabels...)) } } else { for i := range list.Items { resource := list.Items[i] namespace := resource.GetNamespace() name := resource.GetName() debug := debug.WithValues("name", name, "namespace", namespace) if !controllerutils.IsManagedByKyverno(&resource) { var nsLabels map[string]string if namespace != "" { ns, err := h.nsLister.Get(namespace) if err != nil { debug.Error(err, "failed to get namespace labels") errs = append(errs, err) } nsLabels = ns.GetLabels() } // match namespaces if err := match.CheckNamespace(policy.GetNamespace(), resource); err != nil { debug.Info("resource namespace didn't match policy namespace", "result", err) } // match resource with match/exclude clause matched := match.CheckMatchesResources( resource, spec.MatchResources, nsLabels, // TODO(eddycharly): we don't have user info here, we should check that // we don't have user conditions in the policy rule kyvernov1beta1.RequestInfo{}, resource.GroupVersionKind(), "", ) if matched != nil { debug.Info("resource/match didn't match", "result", matched) continue } if spec.ExcludeResources != nil { excluded := match.CheckMatchesResources( resource, *spec.ExcludeResources, nsLabels, // TODO(eddycharly): we don't have user info here, we should check that // we don't have user conditions in the policy rule kyvernov1beta1.RequestInfo{}, resource.GroupVersionKind(), "", ) if excluded == nil { debug.Info("resource/exclude matched") continue } else { debug.Info("resource/exclude didn't match", "result", excluded) } } // check conditions if spec.Conditions != nil { enginectx.Reset() if err := enginectx.SetTargetResource(resource.Object); err != nil { debug.Error(err, "failed to add resource in context") errs = append(errs, err) continue } if err := enginectx.AddNamespace(resource.GetNamespace()); err != nil { debug.Error(err, "failed to add namespace in context") errs = append(errs, err) continue } if err := enginectx.AddImageInfos(&resource, cfg); err != nil { debug.Error(err, "failed to add image infos in context") errs = append(errs, err) continue } passed, err := checkAnyAllConditions(logger, enginectx, *spec.Conditions) if err != nil { debug.Error(err, "failed to check condition") errs = append(errs, err) continue } if !passed { debug.Info("conditions did not pass") continue } } var labels []attribute.KeyValue labels = append(labels, commonLabels...) labels = append(labels, attribute.String("resource_namespace", namespace)) logger.WithValues("name", name, "namespace", namespace).Info("resource matched, it will be deleted...") if err := h.client.DeleteResource(ctx, resource.GetAPIVersion(), resource.GetKind(), namespace, name, false); err != nil { if h.metrics.cleanupFailuresTotal != nil { h.metrics.cleanupFailuresTotal.Add(ctx, 1, metric.WithAttributes(labels...)) } debug.Error(err, "failed to delete resource") errs = append(errs, err) e := event.NewCleanupPolicyEvent(policy, resource, err) h.eventGen.Add(e) } else { if h.metrics.deletedObjectsTotal != nil { h.metrics.deletedObjectsTotal.Add(ctx, 1, metric.WithAttributes(labels...)) } debug.Info("deleted") e := event.NewCleanupPolicyEvent(policy, resource, nil) h.eventGen.Add(e) } } } } } return multierr.Combine(errs...) }