diff --git a/cmd/cleanup-controller/handlers/cleanup/handlers.go b/cmd/cleanup-controller/handlers/cleanup/handlers.go index 709e1322d5..b2dff73a0c 100644 --- a/cmd/cleanup-controller/handlers/cleanup/handlers.go +++ b/cmd/cleanup-controller/handlers/cleanup/handlers.go @@ -12,8 +12,12 @@ import ( "github.com/kyverno/kyverno/pkg/config" enginecontext "github.com/kyverno/kyverno/pkg/engine/context" "github.com/kyverno/kyverno/pkg/event" + "github.com/kyverno/kyverno/pkg/metrics" controllerutils "github.com/kyverno/kyverno/pkg/utils/controller" "github.com/kyverno/kyverno/pkg/utils/match" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric/global" + "go.opentelemetry.io/otel/metric/instrument" "go.uber.org/multierr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -30,6 +34,34 @@ type handlers struct { polLister kyvernov2alpha1listers.CleanupPolicyLister nsLister corev1listers.NamespaceLister recorder record.EventRecorder + metrics cleanupMetrics +} + +type cleanupMetrics struct { + deletedObjectsTotal instrument.Int64Counter + cleanupFailuresTotal instrument.Int64Counter +} + +func newCleanupMetrics(logger logr.Logger) cleanupMetrics { + meter := global.MeterProvider().Meter(metrics.MeterName) + deletedObjectsTotal, err := meter.Int64Counter( + "cleanup_controller_deletedobjects", + instrument.WithDescription("can be used to track number of deleted objects."), + ) + if err != nil { + logger.Error(err, "Failed to create instrument, cleanup_controller_deletedobjects_total") + } + cleanupFailuresTotal, err := meter.Int64Counter( + "cleanup_controller_errors", + instrument.WithDescription("can be used to track number of cleanup failures."), + ) + if err != nil { + logger.Error(err, "Failed to create instrument, cleanup_controller_errors_total") + } + return cleanupMetrics{ + deletedObjectsTotal: deletedObjectsTotal, + cleanupFailuresTotal: cleanupFailuresTotal, + } } func New( @@ -37,6 +69,7 @@ func New( cpolLister kyvernov2alpha1listers.ClusterCleanupPolicyLister, polLister kyvernov2alpha1listers.CleanupPolicyLister, nsLister corev1listers.NamespaceLister, + logger logr.Logger, ) *handlers { return &handlers{ client: client, @@ -44,6 +77,7 @@ func New( polLister: polLister, nsLister: nsLister, recorder: event.NewRecorder(event.CleanupController, client.GetEventsInterface()), + metrics: newCleanupMetrics(logger), } } @@ -75,12 +109,21 @@ func (h *handlers) executePolicy(ctx context.Context, logger logr.Logger, policy debug := logger.V(4) var errs []error for kind := range kinds { + commonLabels := []attribute.KeyValue{ + attribute.String("policy_type", policy.GetKind()), + attribute.String("policy_namespace", policy.GetNamespace()), + attribute.String("policy_name", policy.GetName()), + attribute.String("resource_kind", kind), + } debug := debug.WithValues("kind", kind) debug.Info("processing...") list, err := h.client.ListResource(ctx, "", kind, policy.GetNamespace(), nil) if err != nil { debug.Error(err, "failed to list resources") errs = append(errs, err) + if h.metrics.cleanupFailuresTotal != nil { + h.metrics.cleanupFailuresTotal.Add(ctx, 1, commonLabels...) + } } else { for i := range list.Items { resource := list.Items[i] @@ -165,12 +208,21 @@ func (h *handlers) executePolicy(ctx context.Context, logger logr.Logger, policy continue } } + var labels []attribute.KeyValue + labels = append(labels, commonLabels...) + labels = append(labels, attribute.String("resource_namespace", namespace)) logger.WithValues("name", name, "namespace", namespace).Info("resource matched, it will be deleted...") if err := h.client.DeleteResource(ctx, resource.GetAPIVersion(), resource.GetKind(), namespace, name, false); err != nil { + if h.metrics.cleanupFailuresTotal != nil { + h.metrics.cleanupFailuresTotal.Add(ctx, 1, labels...) + } debug.Error(err, "failed to delete resource") errs = append(errs, err) h.createEvent(policy, resource, err) } else { + if h.metrics.deletedObjectsTotal != nil { + h.metrics.deletedObjectsTotal.Add(ctx, 1, labels...) + } debug.Info("deleted") h.createEvent(policy, resource, nil) } diff --git a/cmd/cleanup-controller/main.go b/cmd/cleanup-controller/main.go index 0d55e0ba1d..617b12bf8a 100644 --- a/cmd/cleanup-controller/main.go +++ b/cmd/cleanup-controller/main.go @@ -193,7 +193,7 @@ func main() { } // create handlers admissionHandlers := admissionhandlers.New(dClient) - cleanupHandlers := cleanuphandlers.New(dClient, cpolLister, polLister, nsLister) + cleanupHandlers := cleanuphandlers.New(dClient, cpolLister, polLister, nsLister, logger.WithName("cleanup-handler")) // create server server := NewServer( func() ([]byte, []byte, error) {