mirrors/kyverno
mirrors/kyverno
1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-03-30 19:35:06 +00:00
Code Activity

feat: added kyverno_policy_rule_execution_latency_milliseconds metric

Browse source 
Signed-off-by: Yashvardhan Kukreja <yash.kukreja.98@gmail.com>
This commit is contained in:
Yashvardhan Kukreja 2021-05-15 21:34:26 +05:30
parent 43a138a12b
commit b8f8a47d8d
8 changed files with 218 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
pkg/metrics/metrics.go
View file

@ -66,8 +66,8 @@ func NewPromConfig() *PromConfig {
}
policyRuleExecutionLatencyMetric := prom.NewGaugeVec(
prom.GaugeOpts{
Name: "kyverno_policy_rule_execution_latency",
Help: "can be used to track the latencies associated with the execution/processing of the individual rules under Kyverno policies whenever they evaluate incoming resource requests.",
Name: "kyverno_policy_rule_execution_latency_milliseconds",
Help: "can be used to track the latencies (in milliseconds) associated with the execution/processing of the individual rules under Kyverno policies whenever they evaluate incoming resource requests.",
},
policyRuleExecutionLatencyLabels,
)

40
pkg/metrics/policyruleexecutionlatency/parsers.go Normal file
View file

@ -0,0 +1,40 @@
package policyruleexecutionlatency
import (
"fmt"
"github.com/kyverno/kyverno/pkg/engine/response"
"github.com/kyverno/kyverno/pkg/metrics"
)
func ParsePromMetrics(pm metrics.PromMetrics) PromMetrics {
return PromMetrics(pm)
}
func ParseRuleTypeFromEngineRuleResponse(rule response.RuleResponse) metrics.RuleType {
switch rule.Type {
case "Validation":
return metrics.Validate
case "Mutation":
return metrics.Mutate
case "Generation":
return metrics.Generate
default:
return metrics.EmptyRuleType
}
}
func ParseResourceRequestOperation(requestOperationStr string) (metrics.ResourceRequestOperation, error) {
switch requestOperationStr {
case "CREATE":
return metrics.ResourceCreated, nil
case "UPDATE":
return metrics.ResourceUpdated, nil
case "DELETE":
return metrics.ResourceDeleted, nil
case "CONNECT":
return metrics.ResourceConnected, nil
default:
return "", fmt.Errorf("Unknown request operation made by resource: %s. Allowed requests: 'CREATE', 'UPDATE', 'DELETE', 'CONNECT'", requestOperationStr)
}
}

117
pkg/metrics/policyruleexecutionlatency/policyRuleExecutionLatency.go Normal file
View file

@ -0,0 +1,117 @@
package policyruleexecutionlatency
import (
"fmt"
"time"
kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
"github.com/kyverno/kyverno/pkg/engine/response"
"github.com/kyverno/kyverno/pkg/metrics"
prom "github.com/prometheus/client_golang/prometheus"
)
func (pm PromMetrics) registerPolicyRuleResultsMetric(
policyValidationMode metrics.PolicyValidationMode,
policyType metrics.PolicyType,
policyBackgroundMode metrics.PolicyBackgroundMode,
policyNamespace, policyName string,
resourceName, resourceKind, resourceNamespace string,
resourceRequestOperation metrics.ResourceRequestOperation,
ruleName string,
ruleResult metrics.RuleResult,
ruleType metrics.RuleType,
ruleExecutionCause metrics.RuleExecutionCause,
ruleResponse string,
mainRequestTriggerTimestamp, policyExecutionTimestamp, ruleExecutionTimestamp int64,
generateRuleLatencyType string,
ruleExecutionLatencyInMs float64,
) error {
if policyType == metrics.Cluster {
policyNamespace = "-"
}
if ruleType != metrics.Generate || generateRuleLatencyType == "" {
generateRuleLatencyType = "-"
}
pm.PolicyRuleExecutionLatency.With(prom.Labels{
"policy_validation_mode": string(policyValidationMode),
"policy_type": string(policyType),
"policy_background_mode": string(policyBackgroundMode),
"policy_namespace": policyNamespace,
"policy_name": policyName,
"resource_name": resourceName,
"resource_kind": resourceKind,
"resource_namespace": resourceNamespace,
"resource_request_operation": string(resourceRequestOperation),
"rule_name": ruleName,
"rule_result": string(ruleResult),
"rule_type": string(ruleType),
"rule_execution_cause": string(ruleExecutionCause),
"rule_response": ruleResponse,
"main_request_trigger_timestamp": fmt.Sprintf("%+v", time.Unix(mainRequestTriggerTimestamp, 0)),
"policy_execution_timestamp": fmt.Sprintf("%+v", time.Unix(policyExecutionTimestamp, 0)),
"rule_execution_timestamp": fmt.Sprintf("%+v", time.Unix(ruleExecutionTimestamp, 0)),
"generate_rule_latency_type": generateRuleLatencyType,
}).Set(ruleExecutionLatencyInMs)
return nil
}
//policy - policy related data
//engineResponse - resource and rule related data
func (pm PromMetrics) ProcessEngineResponse(policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, executionCause metrics.RuleExecutionCause, generateRuleLatencyType string, resourceRequestOperation metrics.ResourceRequestOperation, mainRequestTriggerTimestamp int64) error {
policyValidationMode, err := metrics.ParsePolicyValidationMode(policy.Spec.ValidationFailureAction)
if err != nil {
return err
}
policyType := metrics.Namespaced
policyBackgroundMode := metrics.ParsePolicyBackgroundMode(*policy.Spec.Background)
policyNamespace := policy.ObjectMeta.Namespace
if policyNamespace == "" {
policyNamespace = "-"
policyType = metrics.Cluster
}
policyName := policy.ObjectMeta.Name
policyExecutionTimestamp := engineResponse.PolicyResponse.PolicyExecutionTimestamp
resourceSpec := engineResponse.PolicyResponse.Resource
resourceName := resourceSpec.Name
resourceKind := resourceSpec.Kind
resourceNamespace := resourceSpec.Namespace
ruleResponses := engineResponse.PolicyResponse.Rules
for _, rule := range ruleResponses {
ruleName := rule.Name
ruleType := ParseRuleTypeFromEngineRuleResponse(rule)
ruleResponse := rule.Message
ruleResult := metrics.Fail
if rule.Success {
ruleResult = metrics.Pass
}
ruleExecutionTimestamp := rule.RuleStats.RuleExecutionTimestamp
ruleExecutionLatencyInMs := float64(rule.RuleStats.ProcessingTime) / float64(1000*1000)
if err := pm.registerPolicyRuleResultsMetric(
policyValidationMode,
policyType,
policyBackgroundMode,
policyNamespace, policyName,
resourceName, resourceKind, resourceNamespace,
resourceRequestOperation,
ruleName,
ruleResult,
ruleType,
executionCause,
ruleResponse,
mainRequestTriggerTimestamp, policyExecutionTimestamp, ruleExecutionTimestamp,
generateRuleLatencyType,
ruleExecutionLatencyInMs,
); err != nil {
return err
}
}
return nil
}

7
pkg/metrics/policyruleexecutionlatency/types.go Normal file
View file

@ -0,0 +1,7 @@
package policyruleexecutionlatency
import (
"github.com/kyverno/kyverno/pkg/metrics"
)
type PromMetrics metrics.PromMetrics

9
pkg/policy/existing.go
View file

@ -12,6 +12,7 @@ import (
"github.com/kyverno/kyverno/pkg/engine"
"github.com/kyverno/kyverno/pkg/engine/response"
"github.com/kyverno/kyverno/pkg/metrics"
policyRuleExecutionLatency "github.com/kyverno/kyverno/pkg/metrics/policyruleexecutionlatency"
policyRuleResults "github.com/kyverno/kyverno/pkg/metrics/policyruleresults"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
@ -88,6 +89,8 @@ func (pc *PolicyController) applyAndReportPerNamespace(policy *kyverno.ClusterPo
for _, engineResponse := range engineResponses {
// registering the kyverno_policy_rule_results_info metric concurrently
go pc.registerPolicyRuleResultsMetricValidationBS(logger, *policy, *engineResponse, backgroundScanTimestamp)
// registering the kyverno_policy_rule_execution_latency_milliseconds metric concurrently
go pc.registerPolicyRuleExecutionLatencyMetricValidateBS(logger, *policy, *engineResponse, backgroundScanTimestamp)
}
*metricAlreadyRegistered = true
}
@ -101,6 +104,12 @@ func (pc *PolicyController) registerPolicyRuleResultsMetricValidationBS(logger l
}
}
func (pc *PolicyController) registerPolicyRuleExecutionLatencyMetricValidateBS(logger logr.Logger, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, backgroundScanTimestamp int64) {
if err := policyRuleExecutionLatency.ParsePromMetrics(*pc.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.BackgroundScan, "", metrics.ResourceCreated, backgroundScanTimestamp); err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_execution_latency_milliseconds metrics for the above policy", "name", policy.Name)
}
}
func (pc *PolicyController) applyPolicy(policy *kyverno.ClusterPolicy, resource unstructured.Unstructured, logger logr.Logger) (engineResponses []*response.EngineResponse) {
// pre-processing, check if the policy and resource version has been processed before
if !pc.rm.ProcessResource(policy.Name, policy.ResourceVersion, resource.GetKind(), resource.GetNamespace(), resource.GetName(), resource.GetResourceVersion()) {

14
pkg/webhooks/generation.go
View file

@ -25,6 +25,7 @@ import (
"github.com/kyverno/kyverno/pkg/event"
gen "github.com/kyverno/kyverno/pkg/generate"
"github.com/kyverno/kyverno/pkg/metrics"
policyRuleExecutionLatency "github.com/kyverno/kyverno/pkg/metrics/policyruleexecutionlatency"
policyRuleResults "github.com/kyverno/kyverno/pkg/metrics/policyruleresults"
kyvernoutils "github.com/kyverno/kyverno/pkg/utils"
"github.com/kyverno/kyverno/pkg/webhooks/generate"
@ -85,6 +86,9 @@ func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, polic
}
// registering the kyverno_policy_rule_results_info metric concurrently
go ws.registerPolicyRuleResultsMetricGeneration(logger, string(request.Operation), *policy, *engineResponse, admissionRequestTimestamp)
// registering the kyverno_policy_rule_execution_latency_milliseconds metric concurrently
go ws.registerPolicyRuleExecutionLatencyMetricGenerate(logger, string(request.Operation), *policy, *engineResponse, admissionRequestTimestamp)
}
// Adds Generate Request to a channel(queue size 1000) to generators
@ -112,6 +116,16 @@ func (ws *WebhookServer) registerPolicyRuleResultsMetricGeneration(logger logr.L
}
}
func (ws *WebhookServer) registerPolicyRuleExecutionLatencyMetricGenerate(logger logr.Logger, resourceRequestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, admissionRequestTimestamp int64) {
resourceRequestOperationPromAlias, err := policyRuleExecutionLatency.ParseResourceRequestOperation(resourceRequestOperation)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_execution_latency_milliseconds metrics for the above policy", "name", policy.Name)
}
if err := policyRuleExecutionLatency.ParsePromMetrics(*ws.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, "", resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_execution_latency_milliseconds metrics for the above policy", "name", policy.Name)
}
}
//handleUpdate handles admission-requests for update
func (ws *WebhookServer) handleUpdate(request *v1beta1.AdmissionRequest, policies []*kyverno.ClusterPolicy) {
logger := ws.log.WithValues("action", "generation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation, "gvk", request.Kind.String())

16
pkg/webhooks/mutation.go
View file

@ -15,6 +15,7 @@ import (
"github.com/kyverno/kyverno/pkg/engine/response"
engineutils "github.com/kyverno/kyverno/pkg/engine/utils"
"github.com/kyverno/kyverno/pkg/metrics"
policyRuleExecutionLatency "github.com/kyverno/kyverno/pkg/metrics/policyruleexecutionlatency"
policyRuleResults "github.com/kyverno/kyverno/pkg/metrics/policyruleresults"
v1beta1 "k8s.io/api/admission/v1beta1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@ -43,6 +44,7 @@ func (ws *WebhookServer) HandleMutation(
var patches [][]byte
var engineResponses []*response.EngineResponse
var triggeredPolicies []kyverno.ClusterPolicy
policyContext := &engine.PolicyContext{
NewResource: resource,
AdmissionInfo: userRequestInfo,
@ -94,6 +96,10 @@ func (ws *WebhookServer) HandleMutation(
// registering the kyverno_policy_rule_results_info metric concurrently
go ws.registerPolicyRuleResultsMetricMutation(logger, string(request.Operation), *policy, *engineResponse, admissionRequestTimestamp)
triggeredPolicies = append(triggeredPolicies, *policy)
// registering the kyverno_policy_rule_execution_latency_milliseconds metric concurrently
go ws.registerPolicyRuleExecutionLatencyMetricMutate(logger, string(request.Operation), *policy, *engineResponse, admissionRequestTimestamp)
}
// generate annotations
@ -138,6 +144,16 @@ func (ws *WebhookServer) registerPolicyRuleResultsMetricMutation(logger logr.Log
}
}
func (ws *WebhookServer) registerPolicyRuleExecutionLatencyMetricMutate(logger logr.Logger, resourceRequestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, admissionRequestTimestamp int64) {
resourceRequestOperationPromAlias, err := policyRuleExecutionLatency.ParseResourceRequestOperation(resourceRequestOperation)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_execution_latency_milliseconds metrics for the above policy", "name", policy.Name)
}
if err := policyRuleExecutionLatency.ParsePromMetrics(*ws.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, "", resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_execution_latency_milliseconds metrics for the above policy", "name", policy.Name)
}
}
type mutateStats struct {
resp *response.EngineResponse
namespace string

13
pkg/webhooks/validation.go
View file

@ -24,6 +24,7 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"github.com/kyverno/kyverno/pkg/metrics"
policyRuleExecutionLatency "github.com/kyverno/kyverno/pkg/metrics/policyruleexecutionlatency"
policyRuleResults "github.com/kyverno/kyverno/pkg/metrics/policyruleresults"
)
@ -106,6 +107,8 @@ func HandleValidation(
// registering the kyverno_policy_rule_results_info metric concurrently
go registerPolicyRuleResultsMetricValidation(promConfig, logger, string(request.Operation), policyContext.Policy, *engineResponse, admissionRequestTimestamp)
// registering the kyverno_policy_rule_execution_latency_milliseconds metric concurrently
go registerPolicyRuleExecutionLatencyMetricValidate(promConfig, logger, string(request.Operation), policyContext.Policy, *engineResponse, admissionRequestTimestamp)
engineResponses = append(engineResponses, engineResponse)
statusListener.Update(validateStats{
@ -165,6 +168,16 @@ func registerPolicyRuleResultsMetricValidation(promConfig *metrics.PromConfig, l
}
}
func registerPolicyRuleExecutionLatencyMetricValidate(promConfig *metrics.PromConfig, logger logr.Logger, requestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, admissionRequestTimestamp int64) {
resourceRequestOperationPromAlias, err := policyRuleExecutionLatency.ParseResourceRequestOperation(requestOperation)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_execution_latency_milliseconds metrics for the above policy", "name", policy.Name)
}
if err := policyRuleExecutionLatency.ParsePromMetrics(*promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, "", resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_execution_latency_milliseconds metrics for the above policy", "name", policy.Name)
}
}
func buildDeletionPrInfo(oldR unstructured.Unstructured) policyreport.Info {
return policyreport.Info{
Namespace: oldR.GetNamespace(),