diff --git a/pkg/metrics/admissionreviewlatency/admissionReviewLatency.go b/pkg/metrics/admissionreviewlatency/admissionReviewLatency.go index 7f60fff131..ea8aa6d9a7 100644 --- a/pkg/metrics/admissionreviewlatency/admissionReviewLatency.go +++ b/pkg/metrics/admissionreviewlatency/admissionReviewLatency.go @@ -6,6 +6,7 @@ import ( "github.com/kyverno/kyverno/pkg/engine/response" "github.com/kyverno/kyverno/pkg/metrics" prom "github.com/prometheus/client_golang/prometheus" + "time" ) func (pm PromMetrics) registerAdmissionReviewLatencyMetric( @@ -14,22 +15,24 @@ func (pm PromMetrics) registerAdmissionReviewLatencyMetric( resourceName, resourceKind, resourceNamespace string, resourceRequestOperation metrics.ResourceRequestOperation, admissionRequestLatency float64, + admissionRequestTimestamp int64, ) error { pm.AdmissionReviewLatency.With(prom.Labels{ - "cluster_policies_count": fmt.Sprintf("%d", clusterPoliciesCount), - "namespaced_policies_count": fmt.Sprintf("%d", namespacedPoliciesCount), - "validate_rules_count": fmt.Sprintf("%d", validateRulesCount), - "mutate_rules_count": fmt.Sprintf("%d", mutateRulesCount), - "generate_rules_count": fmt.Sprintf("%d", generateRulesCount), - "resource_name": resourceName, - "resource_kind": resourceKind, - "resource_namespace": resourceNamespace, - "resource_request_operation": string(resourceRequestOperation), + "cluster_policies_count": fmt.Sprintf("%d", clusterPoliciesCount), + "namespaced_policies_count": fmt.Sprintf("%d", namespacedPoliciesCount), + "validate_rules_count": fmt.Sprintf("%d", validateRulesCount), + "mutate_rules_count": fmt.Sprintf("%d", mutateRulesCount), + "generate_rules_count": fmt.Sprintf("%d", generateRulesCount), + "resource_name": resourceName, + "resource_kind": resourceKind, + "resource_namespace": resourceNamespace, + "resource_request_operation": string(resourceRequestOperation), + "admission_request_timestamp": fmt.Sprintf("%+v", time.Unix(admissionRequestTimestamp, 0)), }).Set(admissionRequestLatency) return nil } -func (pm PromMetrics) ProcessEngineResponses(engineResponses []*response.EngineResponse, triggeredPolicies []kyverno.ClusterPolicy, admissionReviewLatencyDuration int64, resourceRequestOperation metrics.ResourceRequestOperation) error { +func (pm PromMetrics) ProcessEngineResponses(engineResponses []*response.EngineResponse, triggeredPolicies []kyverno.ClusterPolicy, admissionReviewLatencyDuration int64, resourceRequestOperation metrics.ResourceRequestOperation, admissionRequestTimestamp int64) error { if len(engineResponses) == 0 { return nil } @@ -64,5 +67,5 @@ func (pm PromMetrics) ProcessEngineResponses(engineResponses []*response.EngineR return nil } admissionReviewLatencyDurationInMs := float64(admissionReviewLatencyDuration) / float64(1000*1000) - return pm.registerAdmissionReviewLatencyMetric(clusterPoliciesCount, namespacedPoliciesCount, totalValidateRulesCount, totalMutateRulesCount, totalGenerateRulesCount, resourceName, resourceKind, resourceNamespace, resourceRequestOperation, admissionReviewLatencyDurationInMs) + return pm.registerAdmissionReviewLatencyMetric(clusterPoliciesCount, namespacedPoliciesCount, totalValidateRulesCount, totalMutateRulesCount, totalGenerateRulesCount, resourceName, resourceKind, resourceNamespace, resourceRequestOperation, admissionReviewLatencyDurationInMs, admissionRequestTimestamp) } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 943c484300..b4f126a087 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -76,6 +76,7 @@ func NewPromConfig() *PromConfig { "cluster_policies_count", "namespaced_policies_count", "validate_rules_count", "mutate_rules_count", "generate_rules_count", "resource_name", "resource_kind", "resource_namespace", "resource_request_operation", + "admission_request_timestamp", } admissionReviewLatencyMetric := prom.NewGaugeVec( prom.GaugeOpts{ diff --git a/pkg/webhooks/generation.go b/pkg/webhooks/generation.go index b84db1957a..8b03e8f9b9 100644 --- a/pkg/webhooks/generation.go +++ b/pkg/webhooks/generation.go @@ -36,10 +36,12 @@ import ( ) //HandleGenerate handles admission-requests for policies with generate rules -func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, policies []*kyverno.ClusterPolicy, ctx *context.Context, userRequestInfo kyverno.RequestInfo, dynamicConfig config.Interface, admissionRequestTimestamp int64, latencySender *chan int64) { +func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, policies []*kyverno.ClusterPolicy, ctx *context.Context, userRequestInfo kyverno.RequestInfo, dynamicConfig config.Interface, admissionRequestTimestamp int64, latencySender *chan int64, triggeredGeneratePoliciesSender *chan []kyverno.ClusterPolicy, generateEngineResponsesSender *chan []*response.EngineResponse) { logger := ws.log.WithValues("action", "generation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation, "gvk", request.Kind.String()) logger.V(4).Info("incoming request") + var engineResponses []*response.EngineResponse + var triggeredGeneratePolicies []kyverno.ClusterPolicy if (request.Operation == v1beta1.Create || request.Operation == v1beta1.Update) && len(policies) != 0 { // convert RAW to unstructured new, old, err := kyvernoutils.ExtractResources(nil, request) @@ -77,6 +79,7 @@ func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, polic engineResponse.PolicyResponse.Rules = rules // some generate rules do apply to the resource engineResponses = append(engineResponses, engineResponse) + triggeredGeneratePolicies = append(triggeredGeneratePolicies, *policy) ws.statusListener.Update(generateStats{ resp: engineResponse, }) @@ -105,6 +108,8 @@ func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, polic // sending the admission request latency to other goroutine (reporting the metrics) over the channel admissionReviewLatencyDuration := int64(time.Since(time.Unix(admissionRequestTimestamp, 0))) *latencySender <- admissionReviewLatencyDuration + *triggeredGeneratePoliciesSender <- triggeredGeneratePolicies + *generateEngineResponsesSender <- engineResponses } func (ws *WebhookServer) registerPolicyRuleResultsMetricGeneration(logger logr.Logger, resourceRequestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, admissionRequestTimestamp int64) { diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index 00dc806572..ba2c374906 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -375,7 +375,7 @@ func (ws *WebhookServer) ResourceMutation(request *v1beta1.AdmissionRequest) *v1 logger.V(6).Info("", "patchedResource", string(patchedResource)) admissionReviewLatencyDuration := int64(time.Since(time.Unix(admissionRequestTimestamp, 0))) // registering the kyverno_admission_review_latency_milliseconds metric concurrently - go registerAdmissionReviewLatencyMetricMutate(logger, *ws.promConfig.Metrics, string(request.Operation), mutateEngineResponses, triggeredMutatePolicies, admissionReviewLatencyDuration) + go registerAdmissionReviewLatencyMetricMutate(logger, *ws.promConfig.Metrics, string(request.Operation), mutateEngineResponses, triggeredMutatePolicies, admissionReviewLatencyDuration, admissionRequestTimestamp) // GENERATE newRequest := request.DeepCopy() @@ -383,11 +383,12 @@ func (ws *WebhookServer) ResourceMutation(request *v1beta1.AdmissionRequest) *v1 // this channel will be used to transmit the admissionReviewLatency from ws.HandleGenerate(..,) goroutine to registeGeneraterPolicyAdmissionReviewLatencyMetric(...) goroutine admissionReviewCompletionLatencyChannel := make(chan int64, 1) + triggeredGeneratePoliciesChannel := make(chan []v1.ClusterPolicy, 1) + generateEngineResponsesChannel := make(chan []*response.EngineResponse, 1) - go ws.HandleGenerate(newRequest, generatePolicies, ctx, userRequestInfo, ws.configHandler, admissionRequestTimestamp, &admissionReviewCompletionLatencyChannel) - + go ws.HandleGenerate(newRequest, generatePolicies, ctx, userRequestInfo, ws.configHandler, admissionRequestTimestamp, &admissionReviewCompletionLatencyChannel, &triggeredGeneratePoliciesChannel, &generateEngineResponsesChannel) // registering the kyverno_admission_review_latency_milliseconds metric concurrently - go registerAdmissionReviewLatencyMetricGenerate(logger, *ws.promConfig.Metrics, string(request.Operation), mutateEngineResponses, triggeredMutatePolicies, &admissionReviewCompletionLatencyChannel) + go registerAdmissionReviewLatencyMetricGenerate(logger, *ws.promConfig.Metrics, string(newRequest.Operation), admissionRequestTimestamp, &admissionReviewCompletionLatencyChannel, &triggeredGeneratePoliciesChannel, &generateEngineResponsesChannel) patchType := v1beta1.PatchTypeJSONPatch return &v1beta1.AdmissionResponse{ Allowed: true, @@ -399,25 +400,31 @@ func (ws *WebhookServer) ResourceMutation(request *v1beta1.AdmissionRequest) *v1 } } -func registerAdmissionReviewLatencyMetricMutate(logger logr.Logger, promMetrics metrics.PromMetrics, requestOperation string, engineResponses []*response.EngineResponse, triggeredPolicies []v1.ClusterPolicy, admissionReviewLatencyDuration int64) { +func registerAdmissionReviewLatencyMetricMutate(logger logr.Logger, promMetrics metrics.PromMetrics, requestOperation string, engineResponses []*response.EngineResponse, triggeredPolicies []v1.ClusterPolicy, admissionReviewLatencyDuration int64, admissionRequestTimestamp int64) { resourceRequestOperationPromAlias, err := admissionReviewLatency.ParseResourceRequestOperation(requestOperation) if err != nil { logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics") } - if err := admissionReviewLatency.ParsePromMetrics(promMetrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias); err != nil { + if err := admissionReviewLatency.ParsePromMetrics(promMetrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil { logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics") } } -func registerAdmissionReviewLatencyMetricGenerate(logger logr.Logger, promMetrics metrics.PromMetrics, requestOperation string, engineResponses []*response.EngineResponse, triggeredPolicies []v1.ClusterPolicy, latencyReceiver *chan int64) { +func registerAdmissionReviewLatencyMetricGenerate(logger logr.Logger, promMetrics metrics.PromMetrics, requestOperation string, admissionRequestTimestamp int64, latencyReceiver *chan int64, triggeredGeneratePoliciesReceiver *chan []v1.ClusterPolicy, engineResponsesReceiver *chan []*response.EngineResponse) { defer close(*latencyReceiver) + defer close(*triggeredGeneratePoliciesReceiver) + defer close(*engineResponsesReceiver) + + triggeredPolicies := <-(*triggeredGeneratePoliciesReceiver) + engineResponses := <-(*engineResponsesReceiver) + resourceRequestOperationPromAlias, err := admissionReviewLatency.ParseResourceRequestOperation(requestOperation) if err != nil { logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics") } // this goroutine will keep on waiting here till it doesn't receive the admission review latency int64 from the other goroutine i.e. ws.HandleGenerate admissionReviewLatencyDuration := <-(*latencyReceiver) - if err := admissionReviewLatency.ParsePromMetrics(promMetrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias); err != nil { + if err := admissionReviewLatency.ParsePromMetrics(promMetrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil { logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics") } } diff --git a/pkg/webhooks/validation.go b/pkg/webhooks/validation.go index 1c3e68b808..902b3deeda 100644 --- a/pkg/webhooks/validation.go +++ b/pkg/webhooks/validation.go @@ -147,7 +147,7 @@ func HandleValidation( logger.V(4).Info("resource blocked") //registering the kyverno_admission_review_latency_milliseconds metric concurrently admissionReviewLatencyDuration := int64(time.Since(time.Unix(admissionRequestTimestamp, 0))) - go registerAdmissionReviewLatencyMetricValidate(promConfig, logger, string(request.Operation), engineResponses, triggeredPolicies, admissionReviewLatencyDuration) + go registerAdmissionReviewLatencyMetricValidate(promConfig, logger, string(request.Operation), engineResponses, triggeredPolicies, admissionReviewLatencyDuration, admissionRequestTimestamp) return false, getEnforceFailureErrorMsg(engineResponses) } @@ -161,7 +161,7 @@ func HandleValidation( //registering the kyverno_admission_review_latency_milliseconds metric concurrently admissionReviewLatencyDuration := int64(time.Since(time.Unix(admissionRequestTimestamp, 0))) - go registerAdmissionReviewLatencyMetricValidate(promConfig, logger, string(request.Operation), engineResponses, triggeredPolicies, admissionReviewLatencyDuration) + go registerAdmissionReviewLatencyMetricValidate(promConfig, logger, string(request.Operation), engineResponses, triggeredPolicies, admissionReviewLatencyDuration, admissionRequestTimestamp) return true, "" } @@ -186,12 +186,12 @@ func registerPolicyRuleExecutionLatencyMetricValidate(promConfig *metrics.PromCo } } -func registerAdmissionReviewLatencyMetricValidate(promConfig *metrics.PromConfig, logger logr.Logger, requestOperation string, engineResponses []*response.EngineResponse, triggeredPolicies []kyverno.ClusterPolicy, admissionReviewLatencyDuration int64) { +func registerAdmissionReviewLatencyMetricValidate(promConfig *metrics.PromConfig, logger logr.Logger, requestOperation string, engineResponses []*response.EngineResponse, triggeredPolicies []kyverno.ClusterPolicy, admissionReviewLatencyDuration int64, admissionRequestTimestamp int64) { resourceRequestOperationPromAlias, err := admissionReviewLatency.ParseResourceRequestOperation(requestOperation) if err != nil { logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics") } - if err := admissionReviewLatency.ParsePromMetrics(*promConfig.Metrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias); err != nil { + if err := admissionReviewLatency.ParsePromMetrics(*promConfig.Metrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil { logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics") } }