feat: add tracing middleware (#5397)

* feat: add tracing middleware

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* add middleware

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* add tracing to middlewares

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Co-authored-by: shuting <shuting@nirmata.com>

cmd/cleanup-controller/server.go

@@ -44,8 +44,11 @@ func NewServer(
 	mux.HandlerFunc(
 		"POST",
 		ValidatingWebhookServicePath,
+		http.HandlerFunc(
 			handlers.AdmissionHandler(policyHandlers.Validate).
-			WithAdmission(logger.Logger.WithName("validate")),
+				WithAdmission(logger.Logger.WithName("validate")).
+				WithTrace(),
+		),
 	)
 	return &server{
 		server: &http.Server{

pkg/tracing/tracing.go

@@ -89,3 +89,17 @@ func DoInSpan(ctx context.Context, tracerName string, operationName string, doFn
 func StartSpan(ctx context.Context, tracerName string, operationName string, attributes ...attribute.KeyValue) (context.Context, trace.Span) {
 	return otel.Tracer(tracerName).Start(ctx, operationName, trace.WithAttributes(attributes...))
 }
+
+// Span executes function doFn inside new span with `operationName` name and hooking as child to a span found within given context if any.
+func Span(ctx context.Context, tracerName string, operationName string, doFn func(context.Context, trace.Span), opts ...trace.SpanStartOption) {
+	newCtx, span := otel.Tracer(tracerName).Start(ctx, operationName, opts...)
+	defer span.End()
+	doFn(newCtx, span)
+}
+
+// Span executes function doFn inside new span with `operationName` name and hooking as child to a span found within given context if any.
+func Span1[T any](ctx context.Context, tracerName string, operationName string, doFn func(context.Context, trace.Span) T, opts ...trace.SpanStartOption) T {
+	newCtx, span := otel.Tracer(tracerName).Start(ctx, operationName, opts...)
+	defer span.End()
+	return doFn(newCtx, span)
+}

pkg/webhooks/handlers/admission.go

@@ -14,13 +14,16 @@ import (
 	admissionv1 "k8s.io/api/admission/v1"
 )
 
-type AdmissionHandler func(context.Context, logr.Logger, *admissionv1.AdmissionRequest, time.Time) *admissionv1.AdmissionResponse
+type (
+	AdmissionHandler func(context.Context, logr.Logger, *admissionv1.AdmissionRequest, time.Time) *admissionv1.AdmissionResponse
+	HttpHandler      func(http.ResponseWriter, *http.Request)
+)
 
-func (h AdmissionHandler) WithAdmission(logger logr.Logger) http.HandlerFunc {
+func (h AdmissionHandler) WithAdmission(logger logr.Logger) HttpHandler {
 	return withAdmission(logger, h)
 }
 
-func withAdmission(logger logr.Logger, inner AdmissionHandler) http.HandlerFunc {
+func withAdmission(logger logr.Logger, inner AdmissionHandler) HttpHandler {
 	return func(writer http.ResponseWriter, request *http.Request) {
 		startTime := time.Now()
 		if request.Body == nil {

pkg/webhooks/handlers/dump.go

@@ -7,7 +7,9 @@ import (
 
 	"github.com/go-logr/logr"
 	engineutils "github.com/kyverno/kyverno/pkg/engine/utils"
+	"github.com/kyverno/kyverno/pkg/tracing"
 	"github.com/kyverno/kyverno/pkg/utils"
+	"go.opentelemetry.io/otel/trace"
 	admissionv1 "k8s.io/api/admission/v1"
 	authenticationv1 "k8s.io/api/authentication/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -103,8 +105,15 @@ func (h AdmissionHandler) WithDump(enabled bool) AdmissionHandler {
 
 func withDump(inner AdmissionHandler) AdmissionHandler {
 	return func(ctx context.Context, logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
+		return tracing.Span1(
+			ctx,
+			"admission_webhook_operations",
+			"dump",
+			func(ctx context.Context, span trace.Span) *admissionv1.AdmissionResponse {
 				response := inner(ctx, logger, request, startTime)
 				dumpPayload(logger, request, response)
 				return response
+			},
+		)
 	}
 }

pkg/webhooks/handlers/filter.go

@@ -6,6 +6,8 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/kyverno/kyverno/pkg/config"
+	"github.com/kyverno/kyverno/pkg/tracing"
+	"go.opentelemetry.io/otel/trace"
 	admissionv1 "k8s.io/api/admission/v1"
 )
 
@@ -15,9 +17,16 @@ func (h AdmissionHandler) WithFilter(configuration config.Configuration) Admissi
 
 func withFilter(c config.Configuration, inner AdmissionHandler) AdmissionHandler {
 	return func(ctx context.Context, logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
+		return tracing.Span1(
+			ctx,
+			"admission_webhook_operations",
+			"filter",
+			func(ctx context.Context, span trace.Span) *admissionv1.AdmissionResponse {
 				if c.ToFilter(request.Kind.Kind, request.Namespace, request.Name) {
 					return nil
 				}
 				return inner(ctx, logger, request, startTime)
+			},
+		)
 	}
 }

pkg/webhooks/handlers/metrics.go

@@ -8,6 +8,8 @@ import (
 	"github.com/kyverno/kyverno/pkg/metrics"
 	admissionRequests "github.com/kyverno/kyverno/pkg/metrics/admissionrequests"
 	admissionReviewDuration "github.com/kyverno/kyverno/pkg/metrics/admissionreviewduration"
+	"github.com/kyverno/kyverno/pkg/tracing"
+	"go.opentelemetry.io/otel/trace"
 	admissionv1 "k8s.io/api/admission/v1"
 )
 
@@ -17,8 +19,15 @@ func (h AdmissionHandler) WithMetrics(metricsConfig *metrics.MetricsConfig) Admi
 
 func withMetrics(metricsConfig *metrics.MetricsConfig, inner AdmissionHandler) AdmissionHandler {
 	return func(ctx context.Context, logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
+		return tracing.Span1(
+			ctx,
+			"admission_webhook_operations",
+			"metrics",
+			func(ctx context.Context, span trace.Span) *admissionv1.AdmissionResponse {
 				defer admissionReviewDuration.Process(metricsConfig, request, int64(time.Since(startTime)))
 				admissionRequests.Process(metricsConfig, request)
 				return inner(ctx, logger, request, startTime)
+			},
+		)
 	}
 }

pkg/webhooks/handlers/protect.go

@@ -9,8 +9,10 @@ import (
 	"github.com/go-logr/logr"
 	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
 	"github.com/kyverno/kyverno/pkg/config"
+	"github.com/kyverno/kyverno/pkg/tracing"
 	"github.com/kyverno/kyverno/pkg/utils"
 	admissionutils "github.com/kyverno/kyverno/pkg/utils/admission"
+	"go.opentelemetry.io/otel/trace"
 	admissionv1 "k8s.io/api/admission/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
@@ -24,6 +26,11 @@ func (h AdmissionHandler) WithProtection(enabled bool) AdmissionHandler {
 
 func withProtection(inner AdmissionHandler) AdmissionHandler {
 	return func(ctx context.Context, logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
+		return tracing.Span1(
+			ctx,
+			"admission_webhook_operations",
+			"protect",
+			func(ctx context.Context, span trace.Span) *admissionv1.AdmissionResponse {
 				newResource, oldResource, err := utils.ExtractResources(nil, request)
 				if err != nil {
 					logger.Error(err, "Failed to extract resources")
@@ -39,5 +46,7 @@ func withProtection(inner AdmissionHandler) AdmissionHandler {
 					}
 				}
 				return inner(ctx, logger, request, startTime)
+			},
+		)
 	}
 }

pkg/webhooks/handlers/trace.go

@@ -0,0 +1,34 @@
+package handlers
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/kyverno/kyverno/pkg/tracing"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+	"go.opentelemetry.io/otel/trace"
+)
+
+func (h HttpHandler) WithTrace() HttpHandler {
+	return withTrace(h)
+}
+
+func withTrace(inner HttpHandler) HttpHandler {
+	return func(writer http.ResponseWriter, request *http.Request) {
+		tracing.Span(
+			request.Context(),
+			"admission_webhook_operations",
+			request.URL.Path,
+			func(ctx context.Context, span trace.Span) {
+				inner(writer, request.WithContext(ctx))
+			},
+			trace.WithAttributes(
+				semconv.HTTPRequestContentLengthKey.Int64(request.ContentLength),
+				semconv.HTTPHostKey.String(request.Host),
+				semconv.HTTPMethodKey.String(request.Method),
+				semconv.HTTPURLKey.String(request.RequestURI),
+			),
+			trace.WithSpanKind(trace.SpanKindServer),
+		)
+	}
+}

pkg/webhooks/handlers/verify.go

@@ -6,13 +6,20 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/kyverno/kyverno/pkg/config"
+	"github.com/kyverno/kyverno/pkg/tracing"
 	admissionutils "github.com/kyverno/kyverno/pkg/utils/admission"
 	jsonutils "github.com/kyverno/kyverno/pkg/utils/json"
+	"go.opentelemetry.io/otel/trace"
 	admissionv1 "k8s.io/api/admission/v1"
 )
 
 func Verify() AdmissionHandler {
 	return func(ctx context.Context, logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
+		return tracing.Span1(
+			ctx,
+			"admission_webhook_operations",
+			"verify",
+			func(ctx context.Context, span trace.Span) *admissionv1.AdmissionResponse {
 				if request.Name != "kyverno-health" || request.Namespace != config.KyvernoNamespace() {
 					return admissionutils.ResponseSuccess()
 				}
@@ -23,5 +30,7 @@ func Verify() AdmissionHandler {
 					return admissionutils.Response(err)
 				}
 				return admissionutils.MutationResponse(bytes)
+			},
+		)
 	}
 }

pkg/webhooks/server.go

@@ -84,26 +84,35 @@ func NewServer(
 	mux.HandlerFunc(
 		"POST",
 		config.PolicyMutatingWebhookServicePath,
+		http.HandlerFunc(
 			handlers.AdmissionHandler(policyHandlers.Mutate).
 				WithFilter(configuration).
 				WithDump(debugModeOpts.DumpPayload).
 				WithMetrics(metricsConfig).
-			WithAdmission(policyLogger.WithName("mutate")),
+				WithAdmission(policyLogger.WithName("mutate")).
+				WithTrace(),
+		),
 	)
 	mux.HandlerFunc(
 		"POST",
 		config.PolicyValidatingWebhookServicePath,
+		http.HandlerFunc(
 			handlers.AdmissionHandler(policyHandlers.Validate).
 				WithFilter(configuration).
 				WithDump(debugModeOpts.DumpPayload).
 				WithMetrics(metricsConfig).
-			WithAdmission(policyLogger.WithName("validate")),
+				WithAdmission(policyLogger.WithName("validate")).
+				WithTrace(),
+		),
 	)
 	mux.HandlerFunc(
 		"POST",
 		config.VerifyMutatingWebhookServicePath,
+		http.HandlerFunc(
 			handlers.Verify().
-			WithAdmission(verifyLogger.WithName("mutate")),
+				WithAdmission(verifyLogger.WithName("mutate")).
+				WithTrace(),
+		),
 	)
 	mux.HandlerFunc("GET", config.LivenessServicePath, handlers.Probe(runtime.IsLive))
 	mux.HandlerFunc("GET", config.ReadinessServicePath, handlers.Probe(runtime.IsReady))
@@ -205,6 +214,7 @@ func registerWebhookHandlers(
 	mux.HandlerFunc(
 		"POST",
 		basePath,
+		http.HandlerFunc(
 			handlers.AdmissionHandler(func(ctx context.Context, logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
 				return handlerFunc(ctx, logger, request, "all", startTime)
 			}).
@@ -212,11 +222,14 @@ func registerWebhookHandlers(
 				WithProtection(toggle.ProtectManagedResources.Enabled()).
 				WithDump(debugModeOpts.DumpPayload).
 				WithMetrics(metricsConfig).
-			WithAdmission(logger),
+				WithAdmission(logger).
+				WithTrace(),
+		),
 	)
 	mux.HandlerFunc(
 		"POST",
 		basePath+"/fail",
+		http.HandlerFunc(
 			handlers.AdmissionHandler(func(ctx context.Context, logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
 				return handlerFunc(ctx, logger, request, "fail", startTime)
 			}).
@@ -224,11 +237,14 @@ func registerWebhookHandlers(
 				WithProtection(toggle.ProtectManagedResources.Enabled()).
 				WithDump(debugModeOpts.DumpPayload).
 				WithMetrics(metricsConfig).
-			WithAdmission(logger),
+				WithAdmission(logger).
+				WithTrace(),
+		),
 	)
 	mux.HandlerFunc(
 		"POST",
 		basePath+"/ignore",
+		http.HandlerFunc(
 			handlers.AdmissionHandler(func(ctx context.Context, logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
 				return handlerFunc(ctx, logger, request, "ignore", startTime)
 			}).
@@ -236,6 +252,8 @@ func registerWebhookHandlers(
 				WithProtection(toggle.ProtectManagedResources.Enabled()).
 				WithDump(debugModeOpts.DumpPayload).
 				WithMetrics(metricsConfig).
-			WithAdmission(logger),
+				WithAdmission(logger).
+				WithTrace(),
+		),
 	)
 }