mirror of
https://github.com/kyverno/kyverno.git
synced 2025-03-31 03:45:17 +00:00
refactor: move metrics closer to the code that use them (#5492)
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
This commit is contained in:
Charles-Edouard Brétéché
GitHub
parent
f9f01fc70d
commit
c2549898c9
5 changed files with 55 additions and 89 deletions
|
|
@ -1,20 +0,0 @@
|
|||
package admissionrequests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/kyverno/kyverno/pkg/metrics"
|
||||
admissionv1 "k8s.io/api/admission/v1"
|
||||
)
|
||||
|
||||
func registerAdmissionRequestsMetric(ctx context.Context, m *metrics.MetricsConfig, resourceKind, resourceNamespace string, resourceRequestOperation metrics.ResourceRequestOperation, allowed bool) {
|
||||
if m.Config.CheckNamespace(resourceNamespace) {
|
||||
m.RecordAdmissionRequests(ctx, resourceKind, resourceNamespace, resourceRequestOperation, allowed)
|
||||
}
|
||||
}
|
||||
|
||||
func Process(ctx context.Context, m *metrics.MetricsConfig, request *admissionv1.AdmissionRequest, response *admissionv1.AdmissionResponse) {
|
||||
op := strings.ToLower(string(request.Operation))
|
||||
registerAdmissionRequestsMetric(ctx, m, request.Kind.Kind, request.Namespace, metrics.ResourceRequestOperation(op), response.Allowed)
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
package admissionreviewduration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/kyverno/kyverno/pkg/metrics"
|
||||
admissionv1 "k8s.io/api/admission/v1"
|
||||
)
|
||||
|
||||
func registerAdmissionReviewDurationMetric(ctx context.Context, m *metrics.MetricsConfig, resourceKind, resourceNamespace string, resourceRequestOperation metrics.ResourceRequestOperation, admissionRequestLatency float64, allowed bool) {
|
||||
if m.Config.CheckNamespace(resourceNamespace) {
|
||||
m.RecordAdmissionReviewDuration(ctx, resourceKind, resourceNamespace, string(resourceRequestOperation), admissionRequestLatency, allowed)
|
||||
}
|
||||
}
|
||||
|
||||
func Process(ctx context.Context, m *metrics.MetricsConfig, request *admissionv1.AdmissionRequest, response *admissionv1.AdmissionResponse, latency int64) {
|
||||
op := strings.ToLower(string(request.Operation))
|
||||
admissionReviewLatencyDurationInSeconds := float64(latency) / float64(1000*1000*1000)
|
||||
registerAdmissionReviewDurationMetric(ctx, m, request.Kind.Kind, request.Namespace, metrics.ResourceRequestOperation(op), admissionReviewLatencyDurationInSeconds, response.Allowed)
|
||||
}
|
||||
|
|
@ -37,8 +37,6 @@ type MetricsConfig struct {
|
|||
policyResultsMetric syncint64.Counter
|
||||
policyRuleInfoMetric asyncfloat64.Gauge
|
||||
policyExecutionDurationMetric syncfloat64.Histogram
|
||||
admissionRequestsMetric syncint64.Counter
|
||||
admissionReviewDurationMetric syncfloat64.Histogram
|
||||
clientQueriesMetric syncint64.Counter
|
||||
|
||||
// config
|
||||
|
|
@ -51,8 +49,6 @@ type MetricsConfigManager interface {
|
|||
RecordPolicyChanges(ctx context.Context, policyValidationMode PolicyValidationMode, policyType PolicyType, policyBackgroundMode PolicyBackgroundMode, policyNamespace string, policyName string, policyChangeType string)
|
||||
RecordPolicyRuleInfo(ctx context.Context, policyValidationMode PolicyValidationMode, policyType PolicyType, policyBackgroundMode PolicyBackgroundMode, policyNamespace string, policyName string, ruleName string, ruleType RuleType, status string, metricValue float64)
|
||||
RecordPolicyExecutionDuration(ctx context.Context, policyValidationMode PolicyValidationMode, policyType PolicyType, policyBackgroundMode PolicyBackgroundMode, policyNamespace string, policyName string, ruleName string, ruleResult RuleResult, ruleType RuleType, ruleExecutionCause RuleExecutionCause, ruleExecutionLatency float64)
|
||||
RecordAdmissionRequests(ctx context.Context, resourceKind string, resourceNamespace string, resourceRequestOperation ResourceRequestOperation, allowed bool)
|
||||
RecordAdmissionReviewDuration(ctx context.Context, resourceKind string, resourceNamespace string, resourceRequestOperation string, admissionRequestLatency float64, allowed bool)
|
||||
RecordClientQueries(ctx context.Context, clientQueryOperation ClientQueryOperation, clientType ClientType, resourceKind string, resourceNamespace string)
|
||||
}
|
||||
|
||||
|
|
@ -72,24 +68,12 @@ func (m *MetricsConfig) initializeMetrics() error {
|
|||
return err
|
||||
}
|
||||
|
||||
m.admissionRequestsMetric, err = meter.SyncInt64().Counter("kyverno_admission_requests_total", instrument.WithDescription("can be used to track the number of admission requests encountered by Kyverno in the cluster"))
|
||||
if err != nil {
|
||||
m.Log.Error(err, "Failed to create instrument, kyverno_admission_requests_total")
|
||||
return err
|
||||
}
|
||||
|
||||
m.policyExecutionDurationMetric, err = meter.SyncFloat64().Histogram("kyverno_policy_execution_duration_seconds", instrument.WithDescription("can be used to track the latencies (in seconds) associated with the execution/processing of the individual rules under Kyverno policies whenever they evaluate incoming resource requests"))
|
||||
if err != nil {
|
||||
m.Log.Error(err, "Failed to create instrument, kyverno_policy_execution_duration_seconds")
|
||||
return err
|
||||
}
|
||||
|
||||
m.admissionReviewDurationMetric, err = meter.SyncFloat64().Histogram("kyverno_admission_review_duration_seconds", instrument.WithDescription("can be used to track the latencies (in seconds) associated with the entire individual admission review. For example, if an incoming request trigger, say, five policies, this metric will track the e2e latency associated with the execution of all those policies"))
|
||||
if err != nil {
|
||||
m.Log.Error(err, "Failed to create instrument, kyverno_admission_review_duration_seconds")
|
||||
return err
|
||||
}
|
||||
|
||||
// Register Async Callbacks
|
||||
m.policyRuleInfoMetric, err = meter.AsyncFloat64().Gauge("kyverno_policy_rule_info_total", instrument.WithDescription("can be used to track the info of the rules or/and policies present in the cluster. 0 means the rule doesn't exist and has been deleted, 1 means the rule is currently existent in the cluster"))
|
||||
if err != nil {
|
||||
|
|
@ -270,16 +254,6 @@ func (m *MetricsConfig) RecordPolicyRuleInfo(ctx context.Context, policyValidati
|
|||
m.policyRuleInfoMetric.Observe(ctx, metricValue, commonLabels...)
|
||||
}
|
||||
|
||||
func (m *MetricsConfig) RecordAdmissionRequests(ctx context.Context, resourceKind string, resourceNamespace string, resourceRequestOperation ResourceRequestOperation, allowed bool) {
|
||||
commonLabels := []attribute.KeyValue{
|
||||
attribute.String("resource_kind", resourceKind),
|
||||
attribute.String("resource_namespace", resourceNamespace),
|
||||
attribute.String("resource_request_operation", string(resourceRequestOperation)),
|
||||
attribute.Bool("request_allowed", allowed),
|
||||
}
|
||||
m.admissionRequestsMetric.Add(ctx, 1, commonLabels...)
|
||||
}
|
||||
|
||||
func (m *MetricsConfig) RecordPolicyExecutionDuration(ctx context.Context, policyValidationMode PolicyValidationMode, policyType PolicyType, policyBackgroundMode PolicyBackgroundMode, policyNamespace string, policyName string,
|
||||
ruleName string, ruleResult RuleResult, ruleType RuleType, ruleExecutionCause RuleExecutionCause, ruleExecutionLatency float64,
|
||||
) {
|
||||
|
|
@ -297,16 +271,6 @@ func (m *MetricsConfig) RecordPolicyExecutionDuration(ctx context.Context, polic
|
|||
m.policyExecutionDurationMetric.Record(ctx, ruleExecutionLatency, commonLabels...)
|
||||
}
|
||||
|
||||
func (m *MetricsConfig) RecordAdmissionReviewDuration(ctx context.Context, resourceKind string, resourceNamespace string, resourceRequestOperation string, admissionRequestLatency float64, allowed bool) {
|
||||
commonLabels := []attribute.KeyValue{
|
||||
attribute.String("resource_kind", resourceKind),
|
||||
attribute.String("resource_namespace", resourceNamespace),
|
||||
attribute.String("resource_request_operation", resourceRequestOperation),
|
||||
attribute.Bool("request_allowed", allowed),
|
||||
}
|
||||
m.admissionReviewDurationMetric.Record(ctx, admissionRequestLatency, commonLabels...)
|
||||
}
|
||||
|
||||
func (m *MetricsConfig) RecordClientQueries(ctx context.Context, clientQueryOperation ClientQueryOperation, clientType ClientType, resourceKind string, resourceNamespace string) {
|
||||
commonLabels := []attribute.KeyValue{
|
||||
attribute.String("operation", string(clientQueryOperation)),
|
||||
|
|
|
|||
|
|
@ -2,24 +2,67 @@ package handlers
|
|||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/kyverno/kyverno/pkg/metrics"
|
||||
admissionRequests "github.com/kyverno/kyverno/pkg/metrics/admissionrequests"
|
||||
admissionReviewDuration "github.com/kyverno/kyverno/pkg/metrics/admissionreviewduration"
|
||||
"github.com/kyverno/kyverno/pkg/config"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric/global"
|
||||
"go.opentelemetry.io/otel/metric/instrument"
|
||||
admissionv1 "k8s.io/api/admission/v1"
|
||||
)
|
||||
|
||||
func (inner AdmissionHandler) WithMetrics(metricsConfig *metrics.MetricsConfig) AdmissionHandler {
|
||||
return inner.withMetrics(metricsConfig).WithTrace("METRICS")
|
||||
func (inner AdmissionHandler) WithMetrics(logger logr.Logger, metricsConfig config.MetricsConfiguration) AdmissionHandler {
|
||||
return inner.withMetrics(logger, metricsConfig).WithTrace("METRICS")
|
||||
}
|
||||
|
||||
func (inner AdmissionHandler) withMetrics(metricsConfig *metrics.MetricsConfig) AdmissionHandler {
|
||||
func (inner AdmissionHandler) withMetrics(logger logr.Logger, metricsConfig config.MetricsConfiguration) AdmissionHandler {
|
||||
meter := global.MeterProvider().Meter("kyverno")
|
||||
admissionRequestsMetric, err := meter.SyncInt64().Counter(
|
||||
"kyverno_admission_requests_total",
|
||||
instrument.WithDescription("can be used to track the number of admission requests encountered by Kyverno in the cluster"),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error(err, "Failed to create instrument, kyverno_admission_requests_total")
|
||||
}
|
||||
admissionReviewDurationMetric, err := meter.SyncFloat64().Histogram(
|
||||
"kyverno_admission_review_duration_seconds",
|
||||
instrument.WithDescription("can be used to track the latencies (in seconds) associated with the entire individual admission review. For example, if an incoming request trigger, say, five policies, this metric will track the e2e latency associated with the execution of all those policies"),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error(err, "Failed to create instrument, kyverno_admission_review_duration_seconds")
|
||||
}
|
||||
return func(ctx context.Context, logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
|
||||
response := inner(ctx, logger, request, startTime)
|
||||
defer admissionReviewDuration.Process(ctx, metricsConfig, request, response, int64(time.Since(startTime)))
|
||||
admissionRequests.Process(ctx, metricsConfig, request, response)
|
||||
namespace := request.Namespace
|
||||
if metricsConfig.CheckNamespace(namespace) {
|
||||
operation := strings.ToLower(string(request.Operation))
|
||||
if admissionReviewDurationMetric != nil {
|
||||
defer func() {
|
||||
latency := int64(time.Since(startTime))
|
||||
admissionReviewLatencyDurationInSeconds := float64(latency) / float64(1000*1000*1000)
|
||||
admissionReviewDurationMetric.Record(
|
||||
ctx,
|
||||
admissionReviewLatencyDurationInSeconds,
|
||||
attribute.String("resource_kind", request.Kind.Kind),
|
||||
attribute.String("resource_namespace", namespace),
|
||||
attribute.String("resource_request_operation", operation),
|
||||
attribute.Bool("request_allowed", response.Allowed),
|
||||
)
|
||||
}()
|
||||
}
|
||||
if admissionRequestsMetric != nil {
|
||||
admissionRequestsMetric.Add(
|
||||
ctx,
|
||||
1,
|
||||
attribute.String("resource_kind", request.Kind.Kind),
|
||||
attribute.String("resource_namespace", namespace),
|
||||
attribute.String("resource_request_operation", operation),
|
||||
attribute.Bool("request_allowed", response.Allowed),
|
||||
)
|
||||
}
|
||||
}
|
||||
return response
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -90,7 +90,7 @@ func NewServer(
|
|||
WithProtection(toggle.ProtectManagedResources.Enabled()).
|
||||
WithDump(debugModeOpts.DumpPayload).
|
||||
WithOperationFilter(admissionv1.Create, admissionv1.Update, admissionv1.Connect).
|
||||
WithMetrics(metricsConfig).
|
||||
WithMetrics(resourceLogger, metricsConfig.Config).
|
||||
WithAdmission(resourceLogger.WithName("mutate"))
|
||||
},
|
||||
)
|
||||
|
|
@ -104,7 +104,7 @@ func NewServer(
|
|||
WithFilter(configuration).
|
||||
WithProtection(toggle.ProtectManagedResources.Enabled()).
|
||||
WithDump(debugModeOpts.DumpPayload).
|
||||
WithMetrics(metricsConfig).
|
||||
WithMetrics(resourceLogger, metricsConfig.Config).
|
||||
WithAdmission(resourceLogger.WithName("validate"))
|
||||
},
|
||||
)
|
||||
|
|
@ -113,7 +113,7 @@ func NewServer(
|
|||
config.PolicyMutatingWebhookServicePath,
|
||||
handlers.FromAdmissionFunc("MUTATE", policyHandlers.Mutate).
|
||||
WithDump(debugModeOpts.DumpPayload).
|
||||
WithMetrics(metricsConfig).
|
||||
WithMetrics(policyLogger, metricsConfig.Config).
|
||||
WithAdmission(policyLogger.WithName("mutate")).
|
||||
ToHandlerFunc(),
|
||||
)
|
||||
|
|
@ -123,7 +123,7 @@ func NewServer(
|
|||
handlers.FromAdmissionFunc("VALIDATE", policyHandlers.Validate).
|
||||
WithDump(debugModeOpts.DumpPayload).
|
||||
WithSubResourceFilter().
|
||||
WithMetrics(metricsConfig).
|
||||
WithMetrics(policyLogger, metricsConfig.Config).
|
||||
WithAdmission(policyLogger.WithName("validate")).
|
||||
ToHandlerFunc(),
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue