diff --git a/pkg/metrics/admissionrequests/admissionRequests.go b/pkg/metrics/admissionrequests/admissionRequests.go deleted file mode 100644 index 4b76090cf2..0000000000 --- a/pkg/metrics/admissionrequests/admissionRequests.go +++ /dev/null @@ -1,20 +0,0 @@ -package admissionrequests - -import ( - "context" - "strings" - - "github.com/kyverno/kyverno/pkg/metrics" - admissionv1 "k8s.io/api/admission/v1" -) - -func registerAdmissionRequestsMetric(ctx context.Context, m *metrics.MetricsConfig, resourceKind, resourceNamespace string, resourceRequestOperation metrics.ResourceRequestOperation, allowed bool) { - if m.Config.CheckNamespace(resourceNamespace) { - m.RecordAdmissionRequests(ctx, resourceKind, resourceNamespace, resourceRequestOperation, allowed) - } -} - -func Process(ctx context.Context, m *metrics.MetricsConfig, request *admissionv1.AdmissionRequest, response *admissionv1.AdmissionResponse) { - op := strings.ToLower(string(request.Operation)) - registerAdmissionRequestsMetric(ctx, m, request.Kind.Kind, request.Namespace, metrics.ResourceRequestOperation(op), response.Allowed) -} diff --git a/pkg/metrics/admissionreviewduration/admissionReviewDuration.go b/pkg/metrics/admissionreviewduration/admissionReviewDuration.go deleted file mode 100644 index 96b31ccf6c..0000000000 --- a/pkg/metrics/admissionreviewduration/admissionReviewDuration.go +++ /dev/null @@ -1,21 +0,0 @@ -package admissionreviewduration - -import ( - "context" - "strings" - - "github.com/kyverno/kyverno/pkg/metrics" - admissionv1 "k8s.io/api/admission/v1" -) - -func registerAdmissionReviewDurationMetric(ctx context.Context, m *metrics.MetricsConfig, resourceKind, resourceNamespace string, resourceRequestOperation metrics.ResourceRequestOperation, admissionRequestLatency float64, allowed bool) { - if m.Config.CheckNamespace(resourceNamespace) { - m.RecordAdmissionReviewDuration(ctx, resourceKind, resourceNamespace, string(resourceRequestOperation), admissionRequestLatency, allowed) - } -} - -func Process(ctx context.Context, m *metrics.MetricsConfig, request *admissionv1.AdmissionRequest, response *admissionv1.AdmissionResponse, latency int64) { - op := strings.ToLower(string(request.Operation)) - admissionReviewLatencyDurationInSeconds := float64(latency) / float64(1000*1000*1000) - registerAdmissionReviewDurationMetric(ctx, m, request.Kind.Kind, request.Namespace, metrics.ResourceRequestOperation(op), admissionReviewLatencyDurationInSeconds, response.Allowed) -} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index c9dca015cb..c97a3c8b60 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -37,8 +37,6 @@ type MetricsConfig struct { policyResultsMetric syncint64.Counter policyRuleInfoMetric asyncfloat64.Gauge policyExecutionDurationMetric syncfloat64.Histogram - admissionRequestsMetric syncint64.Counter - admissionReviewDurationMetric syncfloat64.Histogram clientQueriesMetric syncint64.Counter // config @@ -51,8 +49,6 @@ type MetricsConfigManager interface { RecordPolicyChanges(ctx context.Context, policyValidationMode PolicyValidationMode, policyType PolicyType, policyBackgroundMode PolicyBackgroundMode, policyNamespace string, policyName string, policyChangeType string) RecordPolicyRuleInfo(ctx context.Context, policyValidationMode PolicyValidationMode, policyType PolicyType, policyBackgroundMode PolicyBackgroundMode, policyNamespace string, policyName string, ruleName string, ruleType RuleType, status string, metricValue float64) RecordPolicyExecutionDuration(ctx context.Context, policyValidationMode PolicyValidationMode, policyType PolicyType, policyBackgroundMode PolicyBackgroundMode, policyNamespace string, policyName string, ruleName string, ruleResult RuleResult, ruleType RuleType, ruleExecutionCause RuleExecutionCause, ruleExecutionLatency float64) - RecordAdmissionRequests(ctx context.Context, resourceKind string, resourceNamespace string, resourceRequestOperation ResourceRequestOperation, allowed bool) - RecordAdmissionReviewDuration(ctx context.Context, resourceKind string, resourceNamespace string, resourceRequestOperation string, admissionRequestLatency float64, allowed bool) RecordClientQueries(ctx context.Context, clientQueryOperation ClientQueryOperation, clientType ClientType, resourceKind string, resourceNamespace string) } @@ -72,24 +68,12 @@ func (m *MetricsConfig) initializeMetrics() error { return err } - m.admissionRequestsMetric, err = meter.SyncInt64().Counter("kyverno_admission_requests_total", instrument.WithDescription("can be used to track the number of admission requests encountered by Kyverno in the cluster")) - if err != nil { - m.Log.Error(err, "Failed to create instrument, kyverno_admission_requests_total") - return err - } - m.policyExecutionDurationMetric, err = meter.SyncFloat64().Histogram("kyverno_policy_execution_duration_seconds", instrument.WithDescription("can be used to track the latencies (in seconds) associated with the execution/processing of the individual rules under Kyverno policies whenever they evaluate incoming resource requests")) if err != nil { m.Log.Error(err, "Failed to create instrument, kyverno_policy_execution_duration_seconds") return err } - m.admissionReviewDurationMetric, err = meter.SyncFloat64().Histogram("kyverno_admission_review_duration_seconds", instrument.WithDescription("can be used to track the latencies (in seconds) associated with the entire individual admission review. For example, if an incoming request trigger, say, five policies, this metric will track the e2e latency associated with the execution of all those policies")) - if err != nil { - m.Log.Error(err, "Failed to create instrument, kyverno_admission_review_duration_seconds") - return err - } - // Register Async Callbacks m.policyRuleInfoMetric, err = meter.AsyncFloat64().Gauge("kyverno_policy_rule_info_total", instrument.WithDescription("can be used to track the info of the rules or/and policies present in the cluster. 0 means the rule doesn't exist and has been deleted, 1 means the rule is currently existent in the cluster")) if err != nil { @@ -270,16 +254,6 @@ func (m *MetricsConfig) RecordPolicyRuleInfo(ctx context.Context, policyValidati m.policyRuleInfoMetric.Observe(ctx, metricValue, commonLabels...) } -func (m *MetricsConfig) RecordAdmissionRequests(ctx context.Context, resourceKind string, resourceNamespace string, resourceRequestOperation ResourceRequestOperation, allowed bool) { - commonLabels := []attribute.KeyValue{ - attribute.String("resource_kind", resourceKind), - attribute.String("resource_namespace", resourceNamespace), - attribute.String("resource_request_operation", string(resourceRequestOperation)), - attribute.Bool("request_allowed", allowed), - } - m.admissionRequestsMetric.Add(ctx, 1, commonLabels...) -} - func (m *MetricsConfig) RecordPolicyExecutionDuration(ctx context.Context, policyValidationMode PolicyValidationMode, policyType PolicyType, policyBackgroundMode PolicyBackgroundMode, policyNamespace string, policyName string, ruleName string, ruleResult RuleResult, ruleType RuleType, ruleExecutionCause RuleExecutionCause, ruleExecutionLatency float64, ) { @@ -297,16 +271,6 @@ func (m *MetricsConfig) RecordPolicyExecutionDuration(ctx context.Context, polic m.policyExecutionDurationMetric.Record(ctx, ruleExecutionLatency, commonLabels...) } -func (m *MetricsConfig) RecordAdmissionReviewDuration(ctx context.Context, resourceKind string, resourceNamespace string, resourceRequestOperation string, admissionRequestLatency float64, allowed bool) { - commonLabels := []attribute.KeyValue{ - attribute.String("resource_kind", resourceKind), - attribute.String("resource_namespace", resourceNamespace), - attribute.String("resource_request_operation", resourceRequestOperation), - attribute.Bool("request_allowed", allowed), - } - m.admissionReviewDurationMetric.Record(ctx, admissionRequestLatency, commonLabels...) -} - func (m *MetricsConfig) RecordClientQueries(ctx context.Context, clientQueryOperation ClientQueryOperation, clientType ClientType, resourceKind string, resourceNamespace string) { commonLabels := []attribute.KeyValue{ attribute.String("operation", string(clientQueryOperation)), diff --git a/pkg/webhooks/handlers/metrics.go b/pkg/webhooks/handlers/metrics.go index 0e50a65565..e653f55b62 100644 --- a/pkg/webhooks/handlers/metrics.go +++ b/pkg/webhooks/handlers/metrics.go @@ -2,24 +2,67 @@ package handlers import ( "context" + "strings" "time" "github.com/go-logr/logr" - "github.com/kyverno/kyverno/pkg/metrics" - admissionRequests "github.com/kyverno/kyverno/pkg/metrics/admissionrequests" - admissionReviewDuration "github.com/kyverno/kyverno/pkg/metrics/admissionreviewduration" + "github.com/kyverno/kyverno/pkg/config" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric/global" + "go.opentelemetry.io/otel/metric/instrument" admissionv1 "k8s.io/api/admission/v1" ) -func (inner AdmissionHandler) WithMetrics(metricsConfig *metrics.MetricsConfig) AdmissionHandler { - return inner.withMetrics(metricsConfig).WithTrace("METRICS") +func (inner AdmissionHandler) WithMetrics(logger logr.Logger, metricsConfig config.MetricsConfiguration) AdmissionHandler { + return inner.withMetrics(logger, metricsConfig).WithTrace("METRICS") } -func (inner AdmissionHandler) withMetrics(metricsConfig *metrics.MetricsConfig) AdmissionHandler { +func (inner AdmissionHandler) withMetrics(logger logr.Logger, metricsConfig config.MetricsConfiguration) AdmissionHandler { + meter := global.MeterProvider().Meter("kyverno") + admissionRequestsMetric, err := meter.SyncInt64().Counter( + "kyverno_admission_requests_total", + instrument.WithDescription("can be used to track the number of admission requests encountered by Kyverno in the cluster"), + ) + if err != nil { + logger.Error(err, "Failed to create instrument, kyverno_admission_requests_total") + } + admissionReviewDurationMetric, err := meter.SyncFloat64().Histogram( + "kyverno_admission_review_duration_seconds", + instrument.WithDescription("can be used to track the latencies (in seconds) associated with the entire individual admission review. For example, if an incoming request trigger, say, five policies, this metric will track the e2e latency associated with the execution of all those policies"), + ) + if err != nil { + logger.Error(err, "Failed to create instrument, kyverno_admission_review_duration_seconds") + } return func(ctx context.Context, logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse { response := inner(ctx, logger, request, startTime) - defer admissionReviewDuration.Process(ctx, metricsConfig, request, response, int64(time.Since(startTime))) - admissionRequests.Process(ctx, metricsConfig, request, response) + namespace := request.Namespace + if metricsConfig.CheckNamespace(namespace) { + operation := strings.ToLower(string(request.Operation)) + if admissionReviewDurationMetric != nil { + defer func() { + latency := int64(time.Since(startTime)) + admissionReviewLatencyDurationInSeconds := float64(latency) / float64(1000*1000*1000) + admissionReviewDurationMetric.Record( + ctx, + admissionReviewLatencyDurationInSeconds, + attribute.String("resource_kind", request.Kind.Kind), + attribute.String("resource_namespace", namespace), + attribute.String("resource_request_operation", operation), + attribute.Bool("request_allowed", response.Allowed), + ) + }() + } + if admissionRequestsMetric != nil { + admissionRequestsMetric.Add( + ctx, + 1, + attribute.String("resource_kind", request.Kind.Kind), + attribute.String("resource_namespace", namespace), + attribute.String("resource_request_operation", operation), + attribute.Bool("request_allowed", response.Allowed), + ) + } + } return response } } diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index af8e2b48ee..0670c5fa29 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -90,7 +90,7 @@ func NewServer( WithProtection(toggle.ProtectManagedResources.Enabled()). WithDump(debugModeOpts.DumpPayload). WithOperationFilter(admissionv1.Create, admissionv1.Update, admissionv1.Connect). - WithMetrics(metricsConfig). + WithMetrics(resourceLogger, metricsConfig.Config). WithAdmission(resourceLogger.WithName("mutate")) }, ) @@ -104,7 +104,7 @@ func NewServer( WithFilter(configuration). WithProtection(toggle.ProtectManagedResources.Enabled()). WithDump(debugModeOpts.DumpPayload). - WithMetrics(metricsConfig). + WithMetrics(resourceLogger, metricsConfig.Config). WithAdmission(resourceLogger.WithName("validate")) }, ) @@ -113,7 +113,7 @@ func NewServer( config.PolicyMutatingWebhookServicePath, handlers.FromAdmissionFunc("MUTATE", policyHandlers.Mutate). WithDump(debugModeOpts.DumpPayload). - WithMetrics(metricsConfig). + WithMetrics(policyLogger, metricsConfig.Config). WithAdmission(policyLogger.WithName("mutate")). ToHandlerFunc(), ) @@ -123,7 +123,7 @@ func NewServer( handlers.FromAdmissionFunc("VALIDATE", policyHandlers.Validate). WithDump(debugModeOpts.DumpPayload). WithSubResourceFilter(). - WithMetrics(metricsConfig). + WithMetrics(policyLogger, metricsConfig.Config). WithAdmission(policyLogger.WithName("validate")). ToHandlerFunc(), )