cleanup-controller metrics (#6338)

* cleanup-controller metrics

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>

* add more tags, track errors

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Co-authored-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>
Co-authored-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

cmd/cleanup-controller/handlers/cleanup/handlers.go

@@ -12,8 +12,12 @@ import (
 	"github.com/kyverno/kyverno/pkg/config"
 	enginecontext "github.com/kyverno/kyverno/pkg/engine/context"
 	"github.com/kyverno/kyverno/pkg/event"
+	"github.com/kyverno/kyverno/pkg/metrics"
 	controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
 	"github.com/kyverno/kyverno/pkg/utils/match"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric/global"
+	"go.opentelemetry.io/otel/metric/instrument"
 	"go.uber.org/multierr"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -30,6 +34,34 @@ type handlers struct {
 	polLister  kyvernov2alpha1listers.CleanupPolicyLister
 	nsLister   corev1listers.NamespaceLister
 	recorder   record.EventRecorder
+	metrics    cleanupMetrics
+}
+
+type cleanupMetrics struct {
+	deletedObjectsTotal  instrument.Int64Counter
+	cleanupFailuresTotal instrument.Int64Counter
+}
+
+func newCleanupMetrics(logger logr.Logger) cleanupMetrics {
+	meter := global.MeterProvider().Meter(metrics.MeterName)
+	deletedObjectsTotal, err := meter.Int64Counter(
+		"cleanup_controller_deletedobjects",
+		instrument.WithDescription("can be used to track number of deleted objects."),
+	)
+	if err != nil {
+		logger.Error(err, "Failed to create instrument, cleanup_controller_deletedobjects_total")
+	}
+	cleanupFailuresTotal, err := meter.Int64Counter(
+		"cleanup_controller_errors",
+		instrument.WithDescription("can be used to track number of cleanup failures."),
+	)
+	if err != nil {
+		logger.Error(err, "Failed to create instrument, cleanup_controller_errors_total")
+	}
+	return cleanupMetrics{
+		deletedObjectsTotal:  deletedObjectsTotal,
+		cleanupFailuresTotal: cleanupFailuresTotal,
+	}
 }
 
 func New(
@@ -37,6 +69,7 @@ func New(
 	cpolLister kyvernov2alpha1listers.ClusterCleanupPolicyLister,
 	polLister kyvernov2alpha1listers.CleanupPolicyLister,
 	nsLister corev1listers.NamespaceLister,
+	logger logr.Logger,
 ) *handlers {
 	return &handlers{
 		client:     client,
@@ -44,6 +77,7 @@ func New(
 		polLister:  polLister,
 		nsLister:   nsLister,
 		recorder:   event.NewRecorder(event.CleanupController, client.GetEventsInterface()),
+		metrics:    newCleanupMetrics(logger),
 	}
 }
 
@@ -75,12 +109,21 @@ func (h *handlers) executePolicy(ctx context.Context, logger logr.Logger, policy
 	debug := logger.V(4)
 	var errs []error
 	for kind := range kinds {
+		commonLabels := []attribute.KeyValue{
+			attribute.String("policy_type", policy.GetKind()),
+			attribute.String("policy_namespace", policy.GetNamespace()),
+			attribute.String("policy_name", policy.GetName()),
+			attribute.String("resource_kind", kind),
+		}
 		debug := debug.WithValues("kind", kind)
 		debug.Info("processing...")
 		list, err := h.client.ListResource(ctx, "", kind, policy.GetNamespace(), nil)
 		if err != nil {
 			debug.Error(err, "failed to list resources")
 			errs = append(errs, err)
+			if h.metrics.cleanupFailuresTotal != nil {
+				h.metrics.cleanupFailuresTotal.Add(ctx, 1, commonLabels...)
+			}
 		} else {
 			for i := range list.Items {
 				resource := list.Items[i]
@@ -165,12 +208,21 @@ func (h *handlers) executePolicy(ctx context.Context, logger logr.Logger, policy
 							continue
 						}
 					}
+					var labels []attribute.KeyValue
+					labels = append(labels, commonLabels...)
+					labels = append(labels, attribute.String("resource_namespace", namespace))
 					logger.WithValues("name", name, "namespace", namespace).Info("resource matched, it will be deleted...")
 					if err := h.client.DeleteResource(ctx, resource.GetAPIVersion(), resource.GetKind(), namespace, name, false); err != nil {
+						if h.metrics.cleanupFailuresTotal != nil {
+							h.metrics.cleanupFailuresTotal.Add(ctx, 1, labels...)
+						}
 						debug.Error(err, "failed to delete resource")
 						errs = append(errs, err)
 						h.createEvent(policy, resource, err)
 					} else {
+						if h.metrics.deletedObjectsTotal != nil {
+							h.metrics.deletedObjectsTotal.Add(ctx, 1, labels...)
+						}
 						debug.Info("deleted")
 						h.createEvent(policy, resource, nil)
 					}

cmd/cleanup-controller/main.go

@@ -193,7 +193,7 @@ func main() {
 	}
 	// create handlers
 	admissionHandlers := admissionhandlers.New(dClient)
-	cleanupHandlers := cleanuphandlers.New(dClient, cpolLister, polLister, nsLister)
+	cleanupHandlers := cleanuphandlers.New(dClient, cpolLister, polLister, nsLister, logger.WithName("cleanup-handler"))
 	// create server
 	server := NewServer(
 		func() ([]byte, []byte, error) {