mirrors/kyverno
mirrors/kyverno
1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-03-15 20:20:22 +00:00
Code Activity

dealt with cardinality explosion (#2157)

Browse source
This commit is contained in:
Yashvardhan Kukreja 2021-07-23 21:46:50 +05:30 • committed by GitHub
parent 8c69ce3ea5
commit 0a38f1c8ec
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 429 additions and 350 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
pkg/metrics/admissionrequests/admissionRequests.go Normal file
View file

@ -0,0 +1,53 @@
package admissionrequests
import (
"github.com/kyverno/kyverno/pkg/engine/response"
"github.com/kyverno/kyverno/pkg/metrics"
prom "github.com/prometheus/client_golang/prometheus"
)
func (pm PromMetrics) registerAdmissionRequestsMetric(
resourceName, resourceKind, resourceNamespace string,
resourceRequestOperation metrics.ResourceRequestOperation,
) error {
pm.AdmissionRequests.With(prom.Labels{
"resource_name": resourceName,
"resource_kind": resourceKind,
"resource_namespace": resourceNamespace,
"resource_request_operation": string(resourceRequestOperation),
}).Inc()
return nil
}
func (pm PromMetrics) ProcessEngineResponses(engineResponses []*response.EngineResponse, resourceRequestOperation metrics.ResourceRequestOperation) error {
if len(engineResponses) == 0 {
return nil
}
resourceName, resourceNamespace, resourceKind := engineResponses[0].PolicyResponse.Resource.Name, engineResponses[0].PolicyResponse.Resource.Namespace, engineResponses[0].PolicyResponse.Resource.Kind
totalValidateRulesCount, totalMutateRulesCount, totalGenerateRulesCount := 0, 0, 0
for _, e := range engineResponses {
validateRulesCount, mutateRulesCount, generateRulesCount := 0, 0, 0
for _, rule := range e.PolicyResponse.Rules {
switch rule.Type {
case "Validation":
validateRulesCount++
case "Mutation":
mutateRulesCount++
case "Generation":
generateRulesCount++
}
}
// no rules triggered
if validateRulesCount+mutateRulesCount+generateRulesCount == 0 {
continue
}
totalValidateRulesCount += validateRulesCount
totalMutateRulesCount += mutateRulesCount
totalGenerateRulesCount += generateRulesCount
}
if totalValidateRulesCount+totalMutateRulesCount+totalGenerateRulesCount == 0 {
return nil
}
return pm.registerAdmissionRequestsMetric(resourceName, resourceKind, resourceNamespace, resourceRequestOperation)
}

2
pkg/metrics/admissionreviewlatency/parsers.go → pkg/metrics/admissionrequests/parsers.go
View file

@ -1,4 +1,4 @@
package admissionreviewlatency package admissionrequests
import ( import (
"fmt" "fmt"

2
pkg/metrics/policyruleresults/types.go → pkg/metrics/admissionrequests/types.go
View file

@ -1,4 +1,4 @@
package policyruleresults package admissionrequests
import ( import (
"github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/metrics"

55
pkg/metrics/admissionreviewduration/admissionReviewDuration.go Normal file
View file

@ -0,0 +1,55 @@
package admissionreviewduration
import (
"github.com/kyverno/kyverno/pkg/engine/response"
"github.com/kyverno/kyverno/pkg/metrics"
prom "github.com/prometheus/client_golang/prometheus"
)
func (pm PromMetrics) registerAdmissionReviewDurationMetric(
resourceName, resourceKind, resourceNamespace string,
resourceRequestOperation metrics.ResourceRequestOperation,
admissionRequestLatency float64,
) error {
pm.AdmissionReviewDuration.With(prom.Labels{
"resource_name": resourceName,
"resource_kind": resourceKind,
"resource_namespace": resourceNamespace,
"resource_request_operation": string(resourceRequestOperation),
}).Observe(admissionRequestLatency)
return nil
}
func (pm PromMetrics) ProcessEngineResponses(engineResponses []*response.EngineResponse, admissionReviewLatencyDuration int64, resourceRequestOperation metrics.ResourceRequestOperation) error {
if len(engineResponses) == 0 {
return nil
}
resourceName, resourceNamespace, resourceKind := engineResponses[0].PolicyResponse.Resource.Name, engineResponses[0].PolicyResponse.Resource.Namespace, engineResponses[0].PolicyResponse.Resource.Kind
totalValidateRulesCount, totalMutateRulesCount, totalGenerateRulesCount := 0, 0, 0
for _, e := range engineResponses {
validateRulesCount, mutateRulesCount, generateRulesCount := 0, 0, 0
for _, rule := range e.PolicyResponse.Rules {
switch rule.Type {
case "Validation":
validateRulesCount++
case "Mutation":
mutateRulesCount++
case "Generation":
generateRulesCount++
}
}
// no rules triggered
if validateRulesCount+mutateRulesCount+generateRulesCount == 0 {
continue
}
totalValidateRulesCount += validateRulesCount
totalMutateRulesCount += mutateRulesCount
totalGenerateRulesCount += generateRulesCount
}
if totalValidateRulesCount+totalMutateRulesCount+totalGenerateRulesCount == 0 {
return nil
}
admissionReviewLatencyDurationInSeconds := float64(admissionReviewLatencyDuration) / float64(1000*1000*1000)
return pm.registerAdmissionReviewDurationMetric(resourceName, resourceKind, resourceNamespace, resourceRequestOperation, admissionReviewLatencyDurationInSeconds)
}

26
pkg/metrics/admissionreviewduration/parsers.go Normal file
View file

@ -0,0 +1,26 @@
package admissionreviewduration
import (
"fmt"
"github.com/kyverno/kyverno/pkg/metrics"
)
func ParsePromMetrics(pm metrics.PromMetrics) PromMetrics {
return PromMetrics(pm)
}
func ParseResourceRequestOperation(requestOperationStr string) (metrics.ResourceRequestOperation, error) {
switch requestOperationStr {
case "CREATE":
return metrics.ResourceCreated, nil
case "UPDATE":
return metrics.ResourceUpdated, nil
case "DELETE":
return metrics.ResourceDeleted, nil
case "CONNECT":
return metrics.ResourceConnected, nil
default:
return "", fmt.Errorf("Unknown request operation made by resource: %s. Allowed requests: 'CREATE', 'UPDATE', 'DELETE', 'CONNECT'", requestOperationStr)
}
}

2
pkg/metrics/policyruleexecutionlatency/types.go → pkg/metrics/admissionreviewduration/types.go
View file

@ -1,4 +1,4 @@
package policyruleexecutionlatency package admissionreviewduration
import ( import (
"github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/metrics"

71
pkg/metrics/admissionreviewlatency/admissionReviewLatency.go
View file

@ -1,71 +0,0 @@
package admissionreviewlatency
import (
"fmt"
kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
"github.com/kyverno/kyverno/pkg/engine/response"
"github.com/kyverno/kyverno/pkg/metrics"
prom "github.com/prometheus/client_golang/prometheus"
"time"
)
func (pm PromMetrics) registerAdmissionReviewLatencyMetric(
clusterPoliciesCount, namespacedPoliciesCount int,
validateRulesCount, mutateRulesCount, generateRulesCount int,
resourceName, resourceKind, resourceNamespace string,
resourceRequestOperation metrics.ResourceRequestOperation,
admissionRequestLatency float64,
admissionRequestTimestamp int64,
) error {
pm.AdmissionReviewLatency.With(prom.Labels{
"cluster_policies_count": fmt.Sprintf("%d", clusterPoliciesCount),
"namespaced_policies_count": fmt.Sprintf("%d", namespacedPoliciesCount),
"validate_rules_count": fmt.Sprintf("%d", validateRulesCount),
"mutate_rules_count": fmt.Sprintf("%d", mutateRulesCount),
"generate_rules_count": fmt.Sprintf("%d", generateRulesCount),
"resource_name": resourceName,
"resource_kind": resourceKind,
"resource_namespace": resourceNamespace,
"resource_request_operation": string(resourceRequestOperation),
"admission_request_timestamp": fmt.Sprintf("%+v", time.Unix(admissionRequestTimestamp, 0)),
}).Set(admissionRequestLatency)
return nil
}
func (pm PromMetrics) ProcessEngineResponses(engineResponses []*response.EngineResponse, triggeredPolicies []kyverno.ClusterPolicy, admissionReviewLatencyDuration int64, resourceRequestOperation metrics.ResourceRequestOperation, admissionRequestTimestamp int64) error {
if len(engineResponses) == 0 {
return nil
}
resourceName, resourceNamespace, resourceKind := engineResponses[0].PolicyResponse.Resource.Name, engineResponses[0].PolicyResponse.Resource.Namespace, engineResponses[0].PolicyResponse.Resource.Kind
clusterPoliciesCount, namespacedPoliciesCount, totalValidateRulesCount, totalMutateRulesCount, totalGenerateRulesCount := 0, 0, 0, 0, 0
for i, e := range engineResponses {
validateRulesCount, mutateRulesCount, generateRulesCount := 0, 0, 0
for _, rule := range e.PolicyResponse.Rules {
switch rule.Type {
case "Validation":
validateRulesCount++
case "Mutation":
mutateRulesCount++
case "Generation":
generateRulesCount++
}
}
// no rules triggered
if validateRulesCount+mutateRulesCount+generateRulesCount == 0 {
continue
}
if triggeredPolicies[i].Namespace == "" {
clusterPoliciesCount++
} else {
namespacedPoliciesCount++
}
totalValidateRulesCount += validateRulesCount
totalMutateRulesCount += mutateRulesCount
totalGenerateRulesCount += generateRulesCount
}
if totalValidateRulesCount+totalMutateRulesCount+totalGenerateRulesCount == 0 {
return nil
}
admissionReviewLatencyDurationInMs := float64(admissionReviewLatencyDuration) / float64(1000*1000)
return pm.registerAdmissionReviewLatencyMetric(clusterPoliciesCount, namespacedPoliciesCount, totalValidateRulesCount, totalMutateRulesCount, totalGenerateRulesCount, resourceName, resourceKind, resourceNamespace, resourceRequestOperation, admissionReviewLatencyDurationInMs, admissionRequestTimestamp)
}

93
pkg/metrics/metrics.go
View file

@ -10,11 +10,12 @@ type PromConfig struct {
} }
type PromMetrics struct { type PromMetrics struct {
PolicyRuleResults *prom.GaugeVec PolicyResults *prom.CounterVec
PolicyRuleInfo *prom.GaugeVec PolicyRuleInfo *prom.GaugeVec
PolicyChanges *prom.GaugeVec PolicyChanges *prom.CounterVec
PolicyRuleExecutionLatency *prom.GaugeVec PolicyExecutionDuration *prom.HistogramVec
AdmissionReviewLatency *prom.GaugeVec AdmissionReviewDuration *prom.HistogramVec
AdmissionRequests *prom.CounterVec
} }
func NewPromConfig() *PromConfig { func NewPromConfig() *PromConfig {
@ -22,18 +23,17 @@ func NewPromConfig() *PromConfig {
pc.MetricsRegistry = prom.NewRegistry() pc.MetricsRegistry = prom.NewRegistry()
policyRuleResultsLabels := []string{ policyResultsLabels := []string{
"policy_validation_mode", "policy_type", "policy_background_mode", "policy_name", "policy_namespace", "policy_validation_mode", "policy_type", "policy_background_mode", "policy_name", "policy_namespace",
"resource_name", "resource_kind", "resource_namespace", "resource_request_operation", "resource_name", "resource_kind", "resource_namespace", "resource_request_operation",
"rule_name", "rule_result", "rule_type", "rule_execution_cause", "rule_response", "rule_name", "rule_result", "rule_type", "rule_execution_cause",
"main_request_trigger_timestamp", "policy_execution_timestamp", "rule_execution_timestamp",
} }
policyRuleResultsMetric := prom.NewGaugeVec( policyResultsMetric := prom.NewCounterVec(
prom.GaugeOpts{ prom.CounterOpts{
Name: "kyverno_policy_rule_results_info", Name: "kyverno_policy_results_total",
Help: "can be used to track the results associated with the policies applied in the user’s cluster, at the level from rule to policy to admission requests.", Help: "can be used to track the results associated with the policies applied in the user’s cluster, at the level from rule to policy to admission requests.",
}, },
policyRuleResultsLabels, policyResultsLabels,
) )
policyRuleInfoLabels := []string{ policyRuleInfoLabels := []string{
@ -48,57 +48,66 @@ func NewPromConfig() *PromConfig {
) )
policyChangesLabels := []string{ policyChangesLabels := []string{
"policy_validation_mode", "policy_type", "policy_background_mode", "policy_namespace", "policy_name", "policy_change_type", "timestamp", "policy_validation_mode", "policy_type", "policy_background_mode", "policy_namespace", "policy_name", "policy_change_type",
} }
policyChangesMetric := prom.NewGaugeVec( policyChangesMetric := prom.NewCounterVec(
prom.GaugeOpts{ prom.CounterOpts{
Name: "kyverno_policy_changes_info", Name: "kyverno_policy_changes_total",
Help: "can be used to track all the Kyverno policies which have been created, updated or deleted.", Help: "can be used to track all the changes associated with the Kyverno policies present on the cluster such as creation, updates and deletions.",
}, },
policyChangesLabels, policyChangesLabels,
) )
policyRuleExecutionLatencyLabels := []string{ policyExecutionDurationLabels := []string{
"policy_validation_mode", "policy_type", "policy_background_mode", "policy_name", "policy_namespace", "policy_validation_mode", "policy_type", "policy_background_mode", "policy_name", "policy_namespace",
"resource_name", "resource_kind", "resource_namespace", "resource_request_operation", "resource_name", "resource_kind", "resource_namespace", "resource_request_operation",
"rule_name", "rule_result", "rule_type", "rule_execution_cause", "rule_response", "generate_rule_latency_type", "rule_name", "rule_result", "rule_type", "rule_execution_cause", "generate_rule_latency_type",
"main_request_trigger_timestamp", "policy_execution_timestamp", "rule_execution_timestamp",
} }
policyRuleExecutionLatencyMetric := prom.NewGaugeVec( policyExecutionDurationMetric := prom.NewHistogramVec(
prom.GaugeOpts{ prom.HistogramOpts{
Name: "kyverno_policy_rule_execution_latency_milliseconds", Name: "kyverno_policy_execution_duration_seconds",
Help: "can be used to track the latencies (in milliseconds) associated with the execution/processing of the individual rules under Kyverno policies whenever they evaluate incoming resource requests.", Help: "can be used to track the latencies (in seconds) associated with the execution/processing of the individual rules under Kyverno policies whenever they evaluate incoming resource requests.",
}, },
policyRuleExecutionLatencyLabels, policyExecutionDurationLabels,
) )
admissionReviewLatency := []string{ admissionReviewDurationLabels := []string{
"cluster_policies_count", "namespaced_policies_count",
"validate_rules_count", "mutate_rules_count", "generate_rules_count",
"resource_name", "resource_kind", "resource_namespace", "resource_request_operation", "resource_name", "resource_kind", "resource_namespace", "resource_request_operation",
"admission_request_timestamp",
} }
admissionReviewLatencyMetric := prom.NewGaugeVec( admissionReviewDurationMetric := prom.NewHistogramVec(
prom.GaugeOpts{ prom.HistogramOpts{
Name: "kyverno_admission_review_latency_milliseconds", Name: "kyverno_admission_review_duration_seconds",
Help: "can be used to track the latencies associated with the entire individual admission review. For example, if an incoming request trigger, say, five policies, this metric will track the e2e latency associated with the execution of all those policies.", Help: "can be used to track the latencies (in seconds) associated with the entire individual admission review. For example, if an incoming request trigger, say, five policies, this metric will track the e2e latency associated with the execution of all those policies.",
}, },
admissionReviewLatency, admissionReviewDurationLabels,
)
admissionRequestsLabels := []string{
"resource_name", "resource_kind", "resource_namespace", "resource_request_operation",
}
admissionRequestsMetric := prom.NewCounterVec(
prom.CounterOpts{
Name: "kyverno_admission_requests_total",
Help: "can be used to track the number of admission requests encountered by Kyverno in the cluster.",
},
admissionRequestsLabels,
) )
pc.Metrics = &PromMetrics{ pc.Metrics = &PromMetrics{
PolicyRuleResults: policyRuleResultsMetric, PolicyResults: policyResultsMetric,
PolicyRuleInfo: policyRuleInfoMetric, PolicyRuleInfo: policyRuleInfoMetric,
PolicyChanges: policyChangesMetric, PolicyChanges: policyChangesMetric,
PolicyRuleExecutionLatency: policyRuleExecutionLatencyMetric, PolicyExecutionDuration: policyExecutionDurationMetric,
AdmissionReviewLatency: admissionReviewLatencyMetric, AdmissionReviewDuration: admissionReviewDurationMetric,
AdmissionRequests: admissionRequestsMetric,
} }
pc.MetricsRegistry.MustRegister(pc.Metrics.PolicyRuleResults) pc.MetricsRegistry.MustRegister(pc.Metrics.PolicyResults)
pc.MetricsRegistry.MustRegister(pc.Metrics.PolicyRuleInfo) pc.MetricsRegistry.MustRegister(pc.Metrics.PolicyRuleInfo)
pc.MetricsRegistry.MustRegister(pc.Metrics.PolicyChanges) pc.MetricsRegistry.MustRegister(pc.Metrics.PolicyChanges)
pc.MetricsRegistry.MustRegister(pc.Metrics.PolicyRuleExecutionLatency) pc.MetricsRegistry.MustRegister(pc.Metrics.PolicyExecutionDuration)
pc.MetricsRegistry.MustRegister(pc.Metrics.AdmissionReviewLatency) pc.MetricsRegistry.MustRegister(pc.Metrics.AdmissionReviewDuration)
pc.MetricsRegistry.MustRegister(pc.Metrics.AdmissionRequests)
return pc return pc
} }

11
pkg/metrics/policychanges/policyChanges.go
View file

@ -2,7 +2,6 @@ package policychanges
import ( import (
"fmt" "fmt"
"time"
kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1" kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
"github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/metrics"
@ -15,7 +14,6 @@ func (pm PromMetrics) registerPolicyChangesMetric(
policyBackgroundMode metrics.PolicyBackgroundMode, policyBackgroundMode metrics.PolicyBackgroundMode,
policyNamespace, policyName string, policyNamespace, policyName string,
policyChangeType PolicyChangeType, policyChangeType PolicyChangeType,
policyChangeTimestamp int64,
) error { ) error {
if policyType == metrics.Cluster { if policyType == metrics.Cluster {
policyNamespace = "-" policyNamespace = "-"
@ -27,12 +25,11 @@ func (pm PromMetrics) registerPolicyChangesMetric(
"policy_namespace": policyNamespace, "policy_namespace": policyNamespace,
"policy_name": policyName, "policy_name": policyName,
"policy_change_type": string(policyChangeType), "policy_change_type": string(policyChangeType),
"timestamp": fmt.Sprintf("%+v", time.Unix(policyChangeTimestamp, 0)), }).Inc()
}).Set(1)
return nil return nil
} }
func (pm PromMetrics) RegisterPolicy(policy interface{}, policyChangeType PolicyChangeType, policyChangeTimestamp int64) error { func (pm PromMetrics) RegisterPolicy(policy interface{}, policyChangeType PolicyChangeType) error {
switch inputPolicy := policy.(type) { switch inputPolicy := policy.(type) {
case *kyverno.ClusterPolicy: case *kyverno.ClusterPolicy:
policyValidationMode, err := metrics.ParsePolicyValidationMode(inputPolicy.Spec.ValidationFailureAction) policyValidationMode, err := metrics.ParsePolicyValidationMode(inputPolicy.Spec.ValidationFailureAction)
@ -43,7 +40,7 @@ func (pm PromMetrics) RegisterPolicy(policy interface{}, policyChangeType Policy
policyType := metrics.Cluster policyType := metrics.Cluster
policyNamespace := "" // doesn't matter for cluster policy policyNamespace := "" // doesn't matter for cluster policy
policyName := inputPolicy.ObjectMeta.Name policyName := inputPolicy.ObjectMeta.Name
if err = pm.registerPolicyChangesMetric(policyValidationMode, policyType, policyBackgroundMode, policyNamespace, policyName, policyChangeType, policyChangeTimestamp); err != nil { if err = pm.registerPolicyChangesMetric(policyValidationMode, policyType, policyBackgroundMode, policyNamespace, policyName, policyChangeType); err != nil {
return err return err
} }
return nil return nil
@ -56,7 +53,7 @@ func (pm PromMetrics) RegisterPolicy(policy interface{}, policyChangeType Policy
policyType := metrics.Namespaced policyType := metrics.Namespaced
policyNamespace := inputPolicy.ObjectMeta.Namespace policyNamespace := inputPolicy.ObjectMeta.Namespace
policyName := inputPolicy.ObjectMeta.Name policyName := inputPolicy.ObjectMeta.Name
if err = pm.registerPolicyChangesMetric(policyValidationMode, policyType, policyBackgroundMode, policyNamespace, policyName, policyChangeType, policyChangeTimestamp); err != nil { if err = pm.registerPolicyChangesMetric(policyValidationMode, policyType, policyBackgroundMode, policyNamespace, policyName, policyChangeType); err != nil {
return err return err
} }
return nil return nil

2
pkg/metrics/policyruleexecutionlatency/parsers.go → pkg/metrics/policyexecutionduration/parsers.go
View file

@ -1,4 +1,4 @@
package policyruleexecutionlatency package policyexecutionduration
import ( import (
"fmt" "fmt"

61
pkg/metrics/policyruleexecutionlatency/policyRuleExecutionLatency.go → pkg/metrics/policyexecutionduration/policyExecutionDuration.go
View file

@ -1,16 +1,13 @@
package policyruleexecutionlatency package policyexecutionduration
import ( import (
"fmt"
"time"
kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1" kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
"github.com/kyverno/kyverno/pkg/engine/response" "github.com/kyverno/kyverno/pkg/engine/response"
"github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/metrics"
prom "github.com/prometheus/client_golang/prometheus" prom "github.com/prometheus/client_golang/prometheus"
) )
func (pm PromMetrics) registerPolicyRuleResultsMetric( func (pm PromMetrics) registerPolicyExecutionDurationMetric(
policyValidationMode metrics.PolicyValidationMode, policyValidationMode metrics.PolicyValidationMode,
policyType metrics.PolicyType, policyType metrics.PolicyType,
policyBackgroundMode metrics.PolicyBackgroundMode, policyBackgroundMode metrics.PolicyBackgroundMode,
@ -21,10 +18,8 @@ func (pm PromMetrics) registerPolicyRuleResultsMetric(
ruleResult metrics.RuleResult, ruleResult metrics.RuleResult,
ruleType metrics.RuleType, ruleType metrics.RuleType,
ruleExecutionCause metrics.RuleExecutionCause, ruleExecutionCause metrics.RuleExecutionCause,
ruleResponse string,
mainRequestTriggerTimestamp, policyExecutionTimestamp, ruleExecutionTimestamp int64,
generateRuleLatencyType string, generateRuleLatencyType string,
ruleExecutionLatencyInMs float64, ruleExecutionLatency float64,
) error { ) error {
if policyType == metrics.Cluster { if policyType == metrics.Cluster {
policyNamespace = "-" policyNamespace = "-"
@ -32,32 +27,28 @@ func (pm PromMetrics) registerPolicyRuleResultsMetric(
if ruleType != metrics.Generate || generateRuleLatencyType == "" { if ruleType != metrics.Generate || generateRuleLatencyType == "" {
generateRuleLatencyType = "-" generateRuleLatencyType = "-"
} }
pm.PolicyRuleExecutionLatency.With(prom.Labels{ pm.PolicyExecutionDuration.With(prom.Labels{
"policy_validation_mode": string(policyValidationMode), "policy_validation_mode": string(policyValidationMode),
"policy_type": string(policyType), "policy_type": string(policyType),
"policy_background_mode": string(policyBackgroundMode), "policy_background_mode": string(policyBackgroundMode),
"policy_namespace": policyNamespace, "policy_namespace": policyNamespace,
"policy_name": policyName, "policy_name": policyName,
"resource_name": resourceName, "resource_name": resourceName,
"resource_kind": resourceKind, "resource_kind": resourceKind,
"resource_namespace": resourceNamespace, "resource_namespace": resourceNamespace,
"resource_request_operation": string(resourceRequestOperation), "resource_request_operation": string(resourceRequestOperation),
"rule_name": ruleName, "rule_name": ruleName,
"rule_result": string(ruleResult), "rule_result": string(ruleResult),
"rule_type": string(ruleType), "rule_type": string(ruleType),
"rule_execution_cause": string(ruleExecutionCause), "rule_execution_cause": string(ruleExecutionCause),
"rule_response": ruleResponse, "generate_rule_latency_type": generateRuleLatencyType,
"main_request_trigger_timestamp": fmt.Sprintf("%+v", time.Unix(mainRequestTriggerTimestamp, 0)), }).Observe(ruleExecutionLatency)
"policy_execution_timestamp": fmt.Sprintf("%+v", time.Unix(policyExecutionTimestamp, 0)),
"rule_execution_timestamp": fmt.Sprintf("%+v", time.Unix(ruleExecutionTimestamp, 0)),
"generate_rule_latency_type": generateRuleLatencyType,
}).Set(ruleExecutionLatencyInMs)
return nil return nil
} }
//policy - policy related data //policy - policy related data
//engineResponse - resource and rule related data //engineResponse - resource and rule related data
func (pm PromMetrics) ProcessEngineResponse(policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, executionCause metrics.RuleExecutionCause, generateRuleLatencyType string, resourceRequestOperation metrics.ResourceRequestOperation, mainRequestTriggerTimestamp int64) error { func (pm PromMetrics) ProcessEngineResponse(policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, executionCause metrics.RuleExecutionCause, generateRuleLatencyType string, resourceRequestOperation metrics.ResourceRequestOperation) error {
policyValidationMode, err := metrics.ParsePolicyValidationMode(policy.Spec.ValidationFailureAction) policyValidationMode, err := metrics.ParsePolicyValidationMode(policy.Spec.ValidationFailureAction)
if err != nil { if err != nil {
@ -72,8 +63,6 @@ func (pm PromMetrics) ProcessEngineResponse(policy kyverno.ClusterPolicy, engine
} }
policyName := policy.ObjectMeta.Name policyName := policy.ObjectMeta.Name
policyExecutionTimestamp := engineResponse.PolicyResponse.PolicyExecutionTimestamp
resourceSpec := engineResponse.PolicyResponse.Resource resourceSpec := engineResponse.PolicyResponse.Resource
resourceName := resourceSpec.Name resourceName := resourceSpec.Name
@ -85,16 +74,14 @@ func (pm PromMetrics) ProcessEngineResponse(policy kyverno.ClusterPolicy, engine
for _, rule := range ruleResponses { for _, rule := range ruleResponses {
ruleName := rule.Name ruleName := rule.Name
ruleType := ParseRuleTypeFromEngineRuleResponse(rule) ruleType := ParseRuleTypeFromEngineRuleResponse(rule)
ruleResponse := rule.Message
ruleResult := metrics.Fail ruleResult := metrics.Fail
if rule.Success { if rule.Success {
ruleResult = metrics.Pass ruleResult = metrics.Pass
} }
ruleExecutionTimestamp := rule.RuleStats.RuleExecutionTimestamp ruleExecutionLatencyInSeconds := float64(rule.RuleStats.ProcessingTime) / float64(1000*1000*1000)
ruleExecutionLatencyInMs := float64(rule.RuleStats.ProcessingTime) / float64(1000*1000)
if err := pm.registerPolicyRuleResultsMetric( if err := pm.registerPolicyExecutionDurationMetric(
policyValidationMode, policyValidationMode,
policyType, policyType,
policyBackgroundMode, policyBackgroundMode,
@ -105,10 +92,8 @@ func (pm PromMetrics) ProcessEngineResponse(policy kyverno.ClusterPolicy, engine
ruleResult, ruleResult,
ruleType, ruleType,
executionCause, executionCause,
ruleResponse,
mainRequestTriggerTimestamp, policyExecutionTimestamp, ruleExecutionTimestamp,
generateRuleLatencyType, generateRuleLatencyType,
ruleExecutionLatencyInMs, ruleExecutionLatencyInSeconds,
); err != nil { ); err != nil {
return err return err
} }

7
pkg/metrics/policyexecutionduration/types.go Normal file
View file

@ -0,0 +1,7 @@
package policyexecutionduration
import (
"github.com/kyverno/kyverno/pkg/metrics"
)
type PromMetrics metrics.PromMetrics

3
pkg/metrics/policyruleresults/parsers.go → pkg/metrics/policyresults/parsers.go
View file

@ -1,7 +1,8 @@
package policyruleresults package policyresults
import ( import (
"fmt" "fmt"
"github.com/kyverno/kyverno/pkg/engine/response" "github.com/kyverno/kyverno/pkg/engine/response"
"github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/metrics"
) )

54
pkg/metrics/policyruleresults/policyRuleResults.go → pkg/metrics/policyresults/policyResults.go
View file

@ -1,16 +1,13 @@
package policyruleresults package policyresults
import ( import (
"fmt"
"time"
kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1" kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
"github.com/kyverno/kyverno/pkg/engine/response" "github.com/kyverno/kyverno/pkg/engine/response"
"github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/metrics"
prom "github.com/prometheus/client_golang/prometheus" prom "github.com/prometheus/client_golang/prometheus"
) )
func (pm PromMetrics) registerPolicyRuleResultsMetric( func (pm PromMetrics) registerPolicyResultsMetric(
policyValidationMode metrics.PolicyValidationMode, policyValidationMode metrics.PolicyValidationMode,
policyType metrics.PolicyType, policyType metrics.PolicyType,
policyBackgroundMode metrics.PolicyBackgroundMode, policyBackgroundMode metrics.PolicyBackgroundMode,
@ -21,37 +18,31 @@ func (pm PromMetrics) registerPolicyRuleResultsMetric(
ruleResult metrics.RuleResult, ruleResult metrics.RuleResult,
ruleType metrics.RuleType, ruleType metrics.RuleType,
ruleExecutionCause metrics.RuleExecutionCause, ruleExecutionCause metrics.RuleExecutionCause,
ruleResponse string,
mainRequestTriggerTimestamp, policyExecutionTimestamp, ruleExecutionTimestamp int64,
) error { ) error {
if policyType == metrics.Cluster { if policyType == metrics.Cluster {
policyNamespace = "-" policyNamespace = "-"
} }
pm.PolicyRuleResults.With(prom.Labels{ pm.PolicyResults.With(prom.Labels{
"policy_validation_mode": string(policyValidationMode), "policy_validation_mode": string(policyValidationMode),
"policy_type": string(policyType), "policy_type": string(policyType),
"policy_background_mode": string(policyBackgroundMode), "policy_background_mode": string(policyBackgroundMode),
"policy_namespace": policyNamespace, "policy_namespace": policyNamespace,
"policy_name": policyName, "policy_name": policyName,
"resource_name": resourceName, "resource_name": resourceName,
"resource_kind": resourceKind, "resource_kind": resourceKind,
"resource_namespace": resourceNamespace, "resource_namespace": resourceNamespace,
"resource_request_operation": string(resourceRequestOperation), "resource_request_operation": string(resourceRequestOperation),
"rule_name": ruleName, "rule_name": ruleName,
"rule_result": string(ruleResult), "rule_result": string(ruleResult),
"rule_type": string(ruleType), "rule_type": string(ruleType),
"rule_execution_cause": string(ruleExecutionCause), "rule_execution_cause": string(ruleExecutionCause),
"rule_response": ruleResponse, }).Inc()
"main_request_trigger_timestamp": fmt.Sprintf("%+v", time.Unix(mainRequestTriggerTimestamp, 0)),
"policy_execution_timestamp": fmt.Sprintf("%+v", time.Unix(policyExecutionTimestamp, 0)),
"rule_execution_timestamp": fmt.Sprintf("%+v", time.Unix(ruleExecutionTimestamp, 0)),
}).Set(1)
return nil return nil
} }
//policy - policy related data //policy - policy related data
//engineResponse - resource and rule related data //engineResponse - resource and rule related data
func (pm PromMetrics) ProcessEngineResponse(policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, executionCause metrics.RuleExecutionCause, resourceRequestOperation metrics.ResourceRequestOperation, mainRequestTriggerTimestamp int64) error { func (pm PromMetrics) ProcessEngineResponse(policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, executionCause metrics.RuleExecutionCause, resourceRequestOperation metrics.ResourceRequestOperation) error {
policyValidationMode, err := metrics.ParsePolicyValidationMode(policy.Spec.ValidationFailureAction) policyValidationMode, err := metrics.ParsePolicyValidationMode(policy.Spec.ValidationFailureAction)
if err != nil { if err != nil {
@ -66,8 +57,6 @@ func (pm PromMetrics) ProcessEngineResponse(policy kyverno.ClusterPolicy, engine
} }
policyName := policy.ObjectMeta.Name policyName := policy.ObjectMeta.Name
policyExecutionTimestamp := engineResponse.PolicyResponse.PolicyExecutionTimestamp
resourceSpec := engineResponse.PolicyResponse.Resource resourceSpec := engineResponse.PolicyResponse.Resource
resourceName := resourceSpec.Name resourceName := resourceSpec.Name
@ -79,15 +68,12 @@ func (pm PromMetrics) ProcessEngineResponse(policy kyverno.ClusterPolicy, engine
for _, rule := range ruleResponses { for _, rule := range ruleResponses {
ruleName := rule.Name ruleName := rule.Name
ruleType := ParseRuleTypeFromEngineRuleResponse(rule) ruleType := ParseRuleTypeFromEngineRuleResponse(rule)
ruleResponse := rule.Message
ruleResult := metrics.Fail ruleResult := metrics.Fail
if rule.Success { if rule.Success {
ruleResult = metrics.Pass ruleResult = metrics.Pass
} }
ruleExecutionTimestamp := rule.RuleStats.RuleExecutionTimestamp if err := pm.registerPolicyResultsMetric(
if err := pm.registerPolicyRuleResultsMetric(
policyValidationMode, policyValidationMode,
policyType, policyType,
policyBackgroundMode, policyBackgroundMode,
@ -98,8 +84,6 @@ func (pm PromMetrics) ProcessEngineResponse(policy kyverno.ClusterPolicy, engine
ruleResult, ruleResult,
ruleType, ruleType,
executionCause, executionCause,
ruleResponse,
mainRequestTriggerTimestamp, policyExecutionTimestamp, ruleExecutionTimestamp,
); err != nil { ); err != nil {
return err return err
} }

2
pkg/metrics/admissionreviewlatency/types.go → pkg/metrics/policyresults/types.go
View file

@ -1,4 +1,4 @@
package admissionreviewlatency package policyresults
import ( import (
"github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/metrics"

32
pkg/policy/existing.go
View file

@ -12,12 +12,12 @@ import (
"github.com/kyverno/kyverno/pkg/engine" "github.com/kyverno/kyverno/pkg/engine"
"github.com/kyverno/kyverno/pkg/engine/response" "github.com/kyverno/kyverno/pkg/engine/response"
"github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/metrics"
policyRuleExecutionLatency "github.com/kyverno/kyverno/pkg/metrics/policyruleexecutionlatency" policyExecutionDuration "github.com/kyverno/kyverno/pkg/metrics/policyexecutionduration"
policyRuleResults "github.com/kyverno/kyverno/pkg/metrics/policyruleresults" policyResults "github.com/kyverno/kyverno/pkg/metrics/policyresults"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
) )
func (pc *PolicyController) processExistingResources(policy *kyverno.ClusterPolicy, backgroundScanTimestamp int64) { func (pc *PolicyController) processExistingResources(policy *kyverno.ClusterPolicy) {
logger := pc.log.WithValues("policy", policy.Name) logger := pc.log.WithValues("policy", policy.Name)
logger.V(4).Info("applying policy to existing resources") logger.V(4).Info("applying policy to existing resources")
@ -45,7 +45,7 @@ func (pc *PolicyController) processExistingResources(policy *kyverno.ClusterPoli
metricRegisteredTracker := false metricRegisteredTracker := false
if !namespaced { if !namespaced {
pc.applyAndReportPerNamespace(policy, k, "", rule, logger.WithValues("kind", k), backgroundScanTimestamp, &metricRegisteredTracker) pc.applyAndReportPerNamespace(policy, k, "", rule, logger.WithValues("kind", k), &metricRegisteredTracker)
continue continue
} }
@ -54,7 +54,7 @@ func (pc *PolicyController) processExistingResources(policy *kyverno.ClusterPoli
// for kind: Policy, consider only the namespace which the policy belongs to. // for kind: Policy, consider only the namespace which the policy belongs to.
// for kind: ClusterPolicy, consider all the namespaces. // for kind: ClusterPolicy, consider all the namespaces.
if policy.Namespace == ns || policy.Namespace == "" { if policy.Namespace == ns || policy.Namespace == "" {
pc.applyAndReportPerNamespace(policy, k, ns, rule, logger.WithValues("kind", k).WithValues("ns", ns), backgroundScanTimestamp, &metricRegisteredTracker) pc.applyAndReportPerNamespace(policy, k, ns, rule, logger.WithValues("kind", k).WithValues("ns", ns), &metricRegisteredTracker)
} }
} }
} }
@ -72,7 +72,7 @@ func (pc *PolicyController) registerResource(gvk string) (err error) {
return nil return nil
} }
func (pc *PolicyController) applyAndReportPerNamespace(policy *kyverno.ClusterPolicy, kind string, ns string, rule kyverno.Rule, logger logr.Logger, backgroundScanTimestamp int64, metricAlreadyRegistered *bool) { func (pc *PolicyController) applyAndReportPerNamespace(policy *kyverno.ClusterPolicy, kind string, ns string, rule kyverno.Rule, logger logr.Logger, metricAlreadyRegistered *bool) {
rMap := pc.getResourcesPerNamespace(kind, ns, rule, logger) rMap := pc.getResourcesPerNamespace(kind, ns, rule, logger)
excludeAutoGenResources(*policy, rMap, logger) excludeAutoGenResources(*policy, rMap, logger)
if len(rMap) == 0 { if len(rMap) == 0 {
@ -87,10 +87,10 @@ func (pc *PolicyController) applyAndReportPerNamespace(policy *kyverno.ClusterPo
if !*metricAlreadyRegistered && len(engineResponses) > 0 { if !*metricAlreadyRegistered && len(engineResponses) > 0 {
for _, engineResponse := range engineResponses { for _, engineResponse := range engineResponses {
// registering the kyverno_policy_rule_results_info metric concurrently // registering the kyverno_policy_results_total metric concurrently
go pc.registerPolicyRuleResultsMetricValidationBS(logger, *policy, *engineResponse, backgroundScanTimestamp) go pc.registerPolicyResultsMetricValidation(logger, *policy, *engineResponse)
// registering the kyverno_policy_rule_execution_latency_milliseconds metric concurrently // registering the kyverno_policy_execution_duration_seconds metric concurrently
go pc.registerPolicyRuleExecutionLatencyMetricValidateBS(logger, *policy, *engineResponse, backgroundScanTimestamp) go pc.registerPolicyExecutionDurationMetricValidate(logger, *policy, *engineResponse)
} }
*metricAlreadyRegistered = true *metricAlreadyRegistered = true
} }
@ -98,15 +98,15 @@ func (pc *PolicyController) applyAndReportPerNamespace(policy *kyverno.ClusterPo
pc.report(engineResponses, logger) pc.report(engineResponses, logger)
} }
func (pc *PolicyController) registerPolicyRuleResultsMetricValidationBS(logger logr.Logger, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, backgroundScanTimestamp int64) { func (pc *PolicyController) registerPolicyResultsMetricValidation(logger logr.Logger, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse) {
if err := policyRuleResults.ParsePromMetrics(*pc.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.BackgroundScan, metrics.ResourceCreated, backgroundScanTimestamp); err != nil { if err := policyResults.ParsePromMetrics(*pc.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.BackgroundScan, metrics.ResourceCreated); err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_results_info metrics for the above policy", "name", policy.Name) logger.Error(err, "error occurred while registering kyverno_policy_results_total metrics for the above policy", "name", policy.Name)
} }
} }
func (pc *PolicyController) registerPolicyRuleExecutionLatencyMetricValidateBS(logger logr.Logger, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, backgroundScanTimestamp int64) { func (pc *PolicyController) registerPolicyExecutionDurationMetricValidate(logger logr.Logger, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse) {
if err := policyRuleExecutionLatency.ParsePromMetrics(*pc.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.BackgroundScan, "", metrics.ResourceCreated, backgroundScanTimestamp); err != nil { if err := policyExecutionDuration.ParsePromMetrics(*pc.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.BackgroundScan, "", metrics.ResourceCreated); err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_execution_latency_milliseconds metrics for the above policy", "name", policy.Name) logger.Error(err, "error occurred while registering kyverno_policy_execution_duration_seconds metrics for the above policy", "name", policy.Name)
} }
} }

74
pkg/policy/validate_controller.go
View file

@ -224,34 +224,34 @@ func (pc *PolicyController) registerPolicyRuleInfoMetricDeletePolicy(logger logr
} }
} }
func (pc *PolicyController) registerPolicyChangesMetricAddPolicy(logger logr.Logger, p *kyverno.ClusterPolicy, policyChangeTimestamp int64) { func (pc *PolicyController) registerPolicyChangesMetricAddPolicy(logger logr.Logger, p *kyverno.ClusterPolicy) {
err := policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(p, policyChangesMetric.PolicyCreated, policyChangeTimestamp) err := policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(p, policyChangesMetric.PolicyCreated)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_info metrics for the above policy's creation", "name", p.Name) logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's creation", "name", p.Name)
} }
} }
func (pc *PolicyController) registerPolicyChangesMetricUpdatePolicy(logger logr.Logger, oldP, curP *kyverno.ClusterPolicy, policyChangeTimestamp int64) { func (pc *PolicyController) registerPolicyChangesMetricUpdatePolicy(logger logr.Logger, oldP, curP *kyverno.ClusterPolicy) {
if reflect.DeepEqual((*oldP).Spec, (*curP).Spec) { if reflect.DeepEqual((*oldP).Spec, (*curP).Spec) {
return return
} }
err := policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(oldP, policyChangesMetric.PolicyUpdated, policyChangeTimestamp) err := policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(oldP, policyChangesMetric.PolicyUpdated)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_info metrics for the above policy's updation", "name", oldP.Name) logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's updation", "name", oldP.Name)
} }
// curP will require a new kyverno_policy_changes_info metric if the above update involved change in the following fields: // curP will require a new kyverno_policy_changes_total metric if the above update involved change in the following fields:
if curP.Spec.Background != oldP.Spec.Background || curP.Spec.ValidationFailureAction != oldP.Spec.ValidationFailureAction { if curP.Spec.Background != oldP.Spec.Background || curP.Spec.ValidationFailureAction != oldP.Spec.ValidationFailureAction {
err = policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(curP, policyChangesMetric.PolicyUpdated, policyChangeTimestamp) err = policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(curP, policyChangesMetric.PolicyUpdated)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_info metrics for the above policy's updation", "name", curP.Name) logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's updation", "name", curP.Name)
} }
} }
} }
func (pc *PolicyController) registerPolicyChangesMetricDeletePolicy(logger logr.Logger, p *kyverno.ClusterPolicy, policyChangeTimestamp int64) { func (pc *PolicyController) registerPolicyChangesMetricDeletePolicy(logger logr.Logger, p *kyverno.ClusterPolicy) {
err := policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(p, policyChangesMetric.PolicyDeleted, policyChangeTimestamp) err := policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(p, policyChangesMetric.PolicyDeleted)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_info metrics for the above policy's deletion", "name", p.Name) logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's deletion", "name", p.Name)
} }
} }
@ -263,8 +263,8 @@ func (pc *PolicyController) addPolicy(obj interface{}) {
// register kyverno_policy_rule_info_total metric concurrently // register kyverno_policy_rule_info_total metric concurrently
go pc.registerPolicyRuleInfoMetricAddPolicy(logger, p) go pc.registerPolicyRuleInfoMetricAddPolicy(logger, p)
// register kyverno_policy_changes_info metric concurrently // register kyverno_policy_changes_total metric concurrently
go pc.registerPolicyChangesMetricAddPolicy(logger, p, time.Now().Unix()) go pc.registerPolicyChangesMetricAddPolicy(logger, p)
if p.Spec.Background == nil || p.Spec.ValidationFailureAction == "" || missingAutoGenRules(p, logger) { if p.Spec.Background == nil || p.Spec.ValidationFailureAction == "" || missingAutoGenRules(p, logger) {
pol, _ := common.MutatePolicy(p, logger) pol, _ := common.MutatePolicy(p, logger)
@ -290,8 +290,8 @@ func (pc *PolicyController) updatePolicy(old, cur interface{}) {
// register kyverno_policy_rule_info_total metric concurrently // register kyverno_policy_rule_info_total metric concurrently
go pc.registerPolicyRuleInfoMetricUpdatePolicy(logger, oldP, curP) go pc.registerPolicyRuleInfoMetricUpdatePolicy(logger, oldP, curP)
// register kyverno_policy_changes_info metric concurrently // register kyverno_policy_changes_total metric concurrently
go pc.registerPolicyChangesMetricUpdatePolicy(logger, oldP, curP, time.Now().Unix()) go pc.registerPolicyChangesMetricUpdatePolicy(logger, oldP, curP)
if curP.Spec.Background == nil || curP.Spec.ValidationFailureAction == "" || missingAutoGenRules(curP, logger) { if curP.Spec.Background == nil || curP.Spec.ValidationFailureAction == "" || missingAutoGenRules(curP, logger) {
pol, _ := common.MutatePolicy(curP, logger) pol, _ := common.MutatePolicy(curP, logger)
@ -335,8 +335,8 @@ func (pc *PolicyController) deletePolicy(obj interface{}) {
// register kyverno_policy_rule_info_total metric concurrently // register kyverno_policy_rule_info_total metric concurrently
go pc.registerPolicyRuleInfoMetricDeletePolicy(logger, p) go pc.registerPolicyRuleInfoMetricDeletePolicy(logger, p)
// register kyverno_policy_changes_info metric concurrently // register kyverno_policy_changes_total metric concurrently
go pc.registerPolicyChangesMetricDeletePolicy(logger, p, time.Now().Unix()) go pc.registerPolicyChangesMetricDeletePolicy(logger, p)
logger.Info("policy deleted", "uid", p.UID, "kind", "ClusterPolicy", "name", p.Name) logger.Info("policy deleted", "uid", p.UID, "kind", "ClusterPolicy", "name", p.Name)
@ -380,34 +380,34 @@ func (pc *PolicyController) registerPolicyRuleInfoMetricDeleteNsPolicy(logger lo
} }
} }
func (pc *PolicyController) registerPolicyChangesMetricAddNsPolicy(logger logr.Logger, p *kyverno.Policy, policyChangeTimestamp int64) { func (pc *PolicyController) registerPolicyChangesMetricAddNsPolicy(logger logr.Logger, p *kyverno.Policy) {
err := policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(p, policyChangesMetric.PolicyCreated, policyChangeTimestamp) err := policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(p, policyChangesMetric.PolicyCreated)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_info metrics for the above policy's creation", "name", p.Name) logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's creation", "name", p.Name)
} }
} }
func (pc *PolicyController) registerPolicyChangesMetricUpdateNsPolicy(logger logr.Logger, oldP, curP *kyverno.Policy, policyChangeTimestamp int64) { func (pc *PolicyController) registerPolicyChangesMetricUpdateNsPolicy(logger logr.Logger, oldP, curP *kyverno.Policy) {
if reflect.DeepEqual((*oldP).Spec, (*curP).Spec) { if reflect.DeepEqual((*oldP).Spec, (*curP).Spec) {
return return
} }
err := policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(oldP, policyChangesMetric.PolicyUpdated, policyChangeTimestamp) err := policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(oldP, policyChangesMetric.PolicyUpdated)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_info metrics for the above policy's updation", "name", oldP.Name) logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's updation", "name", oldP.Name)
} }
// curP will require a new kyverno_policy_changes_info metric if the above update involved change in the following fields: // curP will require a new kyverno_policy_changes_total metric if the above update involved change in the following fields:
if curP.Spec.Background != oldP.Spec.Background || curP.Spec.ValidationFailureAction != oldP.Spec.ValidationFailureAction { if curP.Spec.Background != oldP.Spec.Background || curP.Spec.ValidationFailureAction != oldP.Spec.ValidationFailureAction {
err = policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(curP, policyChangesMetric.PolicyUpdated, policyChangeTimestamp) err = policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(curP, policyChangesMetric.PolicyUpdated)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_info metrics for the above policy's updation", "name", curP.Name) logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's updation", "name", curP.Name)
} }
} }
} }
func (pc *PolicyController) registerPolicyChangesMetricDeleteNsPolicy(logger logr.Logger, p *kyverno.Policy, policyChangeTimestamp int64) { func (pc *PolicyController) registerPolicyChangesMetricDeleteNsPolicy(logger logr.Logger, p *kyverno.Policy) {
err := policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(p, policyChangesMetric.PolicyDeleted, policyChangeTimestamp) err := policyChangesMetric.ParsePromMetrics(*pc.promConfig.Metrics).RegisterPolicy(p, policyChangesMetric.PolicyDeleted)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_info metrics for the above policy's deletion", "name", p.Name) logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's deletion", "name", p.Name)
} }
} }
@ -417,8 +417,8 @@ func (pc *PolicyController) addNsPolicy(obj interface{}) {
// register kyverno_policy_rule_info_total metric concurrently // register kyverno_policy_rule_info_total metric concurrently
go pc.registerPolicyRuleInfoMetricAddNsPolicy(logger, p) go pc.registerPolicyRuleInfoMetricAddNsPolicy(logger, p)
// register kyverno_policy_changes_info metric concurrently // register kyverno_policy_changes_total metric concurrently
go pc.registerPolicyChangesMetricAddNsPolicy(logger, p, time.Now().Unix()) go pc.registerPolicyChangesMetricAddNsPolicy(logger, p)
logger.Info("policy created", "uid", p.UID, "kind", "Policy", "name", p.Name, "namespaces", p.Namespace) logger.Info("policy created", "uid", p.UID, "kind", "Policy", "name", p.Name, "namespaces", p.Namespace)
@ -445,8 +445,8 @@ func (pc *PolicyController) updateNsPolicy(old, cur interface{}) {
// register kyverno_policy_rule_info_total metric concurrently // register kyverno_policy_rule_info_total metric concurrently
go pc.registerPolicyRuleInfoMetricUpdateNsPolicy(logger, oldP, curP) go pc.registerPolicyRuleInfoMetricUpdateNsPolicy(logger, oldP, curP)
// register kyverno_policy_changes_info metric concurrently // register kyverno_policy_changes_total metric concurrently
go pc.registerPolicyChangesMetricUpdateNsPolicy(logger, oldP, curP, time.Now().Unix()) go pc.registerPolicyChangesMetricUpdateNsPolicy(logger, oldP, curP)
ncurP := ConvertPolicyToClusterPolicy(curP) ncurP := ConvertPolicyToClusterPolicy(curP)
@ -492,8 +492,8 @@ func (pc *PolicyController) deleteNsPolicy(obj interface{}) {
// register kyverno_policy_rule_info_total metric concurrently // register kyverno_policy_rule_info_total metric concurrently
go pc.registerPolicyRuleInfoMetricDeleteNsPolicy(logger, p) go pc.registerPolicyRuleInfoMetricDeleteNsPolicy(logger, p)
// register kyverno_policy_changes_info metric concurrently // register kyverno_policy_changes_total metric concurrently
go pc.registerPolicyChangesMetricDeleteNsPolicy(logger, p, time.Now().Unix()) go pc.registerPolicyChangesMetricDeleteNsPolicy(logger, p)
logger.Info("policy deleted event", "uid", p.UID, "kind", "Policy", "policy_name", p.Name, "namespaces", p.Namespace) logger.Info("policy deleted event", "uid", p.UID, "kind", "Policy", "policy_name", p.Name, "namespaces", p.Namespace)
@ -640,7 +640,7 @@ func (pc *PolicyController) syncPolicy(key string) error {
} }
updateGR(pc.kyvernoClient, policy.Name, grList, logger) updateGR(pc.kyvernoClient, policy.Name, grList, logger)
pc.processExistingResources(policy, startTime.Unix()) pc.processExistingResources(policy)
return nil return nil
} }

53
pkg/webhooks/generation.go
View file

@ -23,8 +23,8 @@ import (
"github.com/kyverno/kyverno/pkg/event" "github.com/kyverno/kyverno/pkg/event"
gen "github.com/kyverno/kyverno/pkg/generate" gen "github.com/kyverno/kyverno/pkg/generate"
"github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/metrics"
policyRuleExecutionLatency "github.com/kyverno/kyverno/pkg/metrics/policyruleexecutionlatency" policyExecutionDuration "github.com/kyverno/kyverno/pkg/metrics/policyexecutionduration"
policyRuleResults "github.com/kyverno/kyverno/pkg/metrics/policyruleresults" policyResults "github.com/kyverno/kyverno/pkg/metrics/policyresults"
kyvernoutils "github.com/kyverno/kyverno/pkg/utils" kyvernoutils "github.com/kyverno/kyverno/pkg/utils"
"github.com/kyverno/kyverno/pkg/webhooks/generate" "github.com/kyverno/kyverno/pkg/webhooks/generate"
v1beta1 "k8s.io/api/admission/v1beta1" v1beta1 "k8s.io/api/admission/v1beta1"
@ -35,10 +35,12 @@ import (
func (ws *WebhookServer) applyGeneratePolicies(request *v1beta1.AdmissionRequest, policyContext *engine.PolicyContext, policies []*v1.ClusterPolicy, ts int64, logger logr.Logger) { func (ws *WebhookServer) applyGeneratePolicies(request *v1beta1.AdmissionRequest, policyContext *engine.PolicyContext, policies []*v1.ClusterPolicy, ts int64, logger logr.Logger) {
admissionReviewCompletionLatencyChannel := make(chan int64, 1) admissionReviewCompletionLatencyChannel := make(chan int64, 1)
triggeredGeneratePoliciesChannel := make(chan []v1.ClusterPolicy, 1) generateEngineResponsesSenderForAdmissionReviewDurationMetric := make(chan []*response.EngineResponse, 1)
generateEngineResponsesChannel := make(chan []*response.EngineResponse, 1) generateEngineResponsesSenderForAdmissionRequestsCountMetric := make(chan []*response.EngineResponse, 1)
go ws.handleGenerate(request, policies, policyContext.JSONContext, policyContext.AdmissionInfo, ws.configHandler, ts, &admissionReviewCompletionLatencyChannel, &triggeredGeneratePoliciesChannel, &generateEngineResponsesChannel)
go registerAdmissionReviewLatencyMetricGenerate(logger, *ws.promConfig.Metrics, string(request.Operation), ts, &admissionReviewCompletionLatencyChannel, &triggeredGeneratePoliciesChannel, &generateEngineResponsesChannel) go ws.handleGenerate(request, policies, policyContext.JSONContext, policyContext.AdmissionInfo, ws.configHandler, ts, &admissionReviewCompletionLatencyChannel, &generateEngineResponsesSenderForAdmissionReviewDurationMetric, &generateEngineResponsesSenderForAdmissionRequestsCountMetric)
go registerAdmissionReviewDurationMetricGenerate(logger, *ws.promConfig.Metrics, string(request.Operation), &admissionReviewCompletionLatencyChannel, &generateEngineResponsesSenderForAdmissionReviewDurationMetric)
go registerAdmissionRequestsMetricGenerate(logger, *ws.promConfig.Metrics, string(request.Operation), &generateEngineResponsesSenderForAdmissionRequestsCountMetric)
} }
//handleGenerate handles admission-requests for policies with generate rules //handleGenerate handles admission-requests for policies with generate rules
@ -50,14 +52,14 @@ func (ws *WebhookServer) handleGenerate(
dynamicConfig config.Interface, dynamicConfig config.Interface,
admissionRequestTimestamp int64, admissionRequestTimestamp int64,
latencySender *chan int64, latencySender *chan int64,
triggeredGeneratePoliciesSender *chan []kyverno.ClusterPolicy, generateEngineResponsesSenderForAdmissionReviewDurationMetric *chan []*response.EngineResponse,
generateEngineResponsesSender *chan []*response.EngineResponse) { generateEngineResponsesSenderForAdmissionRequestsCountMetric *chan []*response.EngineResponse,
) {
logger := ws.log.WithValues("action", "generation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation, "gvk", request.Kind.String()) logger := ws.log.WithValues("action", "generation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation, "gvk", request.Kind.String())
logger.V(6).Info("generate request") logger.V(6).Info("generate request")
var engineResponses []*response.EngineResponse var engineResponses []*response.EngineResponse
var triggeredGeneratePolicies []kyverno.ClusterPolicy
if (request.Operation == v1beta1.Create || request.Operation == v1beta1.Update) && len(policies) != 0 { if (request.Operation == v1beta1.Create || request.Operation == v1beta1.Update) && len(policies) != 0 {
// convert RAW to unstructured // convert RAW to unstructured
new, old, err := kyvernoutils.ExtractResources(nil, request) new, old, err := kyvernoutils.ExtractResources(nil, request)
@ -95,14 +97,13 @@ func (ws *WebhookServer) handleGenerate(
engineResponse.PolicyResponse.Rules = rules engineResponse.PolicyResponse.Rules = rules
// some generate rules do apply to the resource // some generate rules do apply to the resource
engineResponses = append(engineResponses, engineResponse) engineResponses = append(engineResponses, engineResponse)
triggeredGeneratePolicies = append(triggeredGeneratePolicies, *policy)
} }
// registering the kyverno_policy_rule_results_info metric concurrently // registering the kyverno_policy_results_total metric concurrently
go ws.registerPolicyRuleResultsMetricGeneration(logger, string(request.Operation), *policy, *engineResponse, admissionRequestTimestamp) go ws.registerPolicyResultsMetricGeneration(logger, string(request.Operation), *policy, *engineResponse)
// registering the kyverno_policy_rule_execution_latency_milliseconds metric concurrently // registering the kyverno_policy_execution_duration_seconds metric concurrently
go ws.registerPolicyRuleExecutionLatencyMetricGenerate(logger, string(request.Operation), *policy, *engineResponse, admissionRequestTimestamp) go ws.registerPolicyExecutionDurationMetricGenerate(logger, string(request.Operation), *policy, *engineResponse)
} }
// Adds Generate Request to a channel(queue size 1000) to generators // Adds Generate Request to a channel(queue size 1000) to generators
@ -122,27 +123,27 @@ func (ws *WebhookServer) handleGenerate(
// sending the admission request latency to other goroutine (reporting the metrics) over the channel // sending the admission request latency to other goroutine (reporting the metrics) over the channel
admissionReviewLatencyDuration := int64(time.Since(time.Unix(admissionRequestTimestamp, 0))) admissionReviewLatencyDuration := int64(time.Since(time.Unix(admissionRequestTimestamp, 0)))
*latencySender <- admissionReviewLatencyDuration *latencySender <- admissionReviewLatencyDuration
*triggeredGeneratePoliciesSender <- triggeredGeneratePolicies *generateEngineResponsesSenderForAdmissionReviewDurationMetric <- engineResponses
*generateEngineResponsesSender <- engineResponses *generateEngineResponsesSenderForAdmissionRequestsCountMetric <- engineResponses
} }
func (ws *WebhookServer) registerPolicyRuleResultsMetricGeneration(logger logr.Logger, resourceRequestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, admissionRequestTimestamp int64) { func (ws *WebhookServer) registerPolicyResultsMetricGeneration(logger logr.Logger, resourceRequestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse) {
resourceRequestOperationPromAlias, err := policyRuleResults.ParseResourceRequestOperation(resourceRequestOperation) resourceRequestOperationPromAlias, err := policyResults.ParseResourceRequestOperation(resourceRequestOperation)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_results_info metrics for the above policy", "name", policy.Name) logger.Error(err, "error occurred while registering kyverno_policy_results_total metrics for the above policy", "name", policy.Name)
} }
if err := policyRuleResults.ParsePromMetrics(*ws.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil { if err := policyResults.ParsePromMetrics(*ws.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, resourceRequestOperationPromAlias); err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_results_info metrics for the above policy", "name", policy.Name) logger.Error(err, "error occurred while registering kyverno_policy_results_total metrics for the above policy", "name", policy.Name)
} }
} }
func (ws *WebhookServer) registerPolicyRuleExecutionLatencyMetricGenerate(logger logr.Logger, resourceRequestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, admissionRequestTimestamp int64) { func (ws *WebhookServer) registerPolicyExecutionDurationMetricGenerate(logger logr.Logger, resourceRequestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse) {
resourceRequestOperationPromAlias, err := policyRuleExecutionLatency.ParseResourceRequestOperation(resourceRequestOperation) resourceRequestOperationPromAlias, err := policyExecutionDuration.ParseResourceRequestOperation(resourceRequestOperation)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_execution_latency_milliseconds metrics for the above policy", "name", policy.Name) logger.Error(err, "error occurred while registering kyverno_policy_execution_duration_seconds metrics for the above policy", "name", policy.Name)
} }
if err := policyRuleExecutionLatency.ParsePromMetrics(*ws.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, "", resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil { if err := policyExecutionDuration.ParsePromMetrics(*ws.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, "", resourceRequestOperationPromAlias); err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_execution_latency_milliseconds metrics for the above policy", "name", policy.Name) logger.Error(err, "error occurred while registering kyverno_policy_execution_duration_seconds metrics for the above policy", "name", policy.Name)
} }
} }

52
pkg/webhooks/mutation.go
View file

@ -14,8 +14,8 @@ import (
"github.com/kyverno/kyverno/pkg/engine/response" "github.com/kyverno/kyverno/pkg/engine/response"
engineutils "github.com/kyverno/kyverno/pkg/engine/utils" engineutils "github.com/kyverno/kyverno/pkg/engine/utils"
"github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/metrics"
policyRuleExecutionLatency "github.com/kyverno/kyverno/pkg/metrics/policyruleexecutionlatency" policyExecutionDuration "github.com/kyverno/kyverno/pkg/metrics/policyexecutionduration"
policyRuleResults "github.com/kyverno/kyverno/pkg/metrics/policyruleresults" policyResults "github.com/kyverno/kyverno/pkg/metrics/policyresults"
"github.com/kyverno/kyverno/pkg/utils" "github.com/kyverno/kyverno/pkg/utils"
"github.com/pkg/errors" "github.com/pkg/errors"
v1beta1 "k8s.io/api/admission/v1beta1" v1beta1 "k8s.io/api/admission/v1beta1"
@ -24,14 +24,14 @@ import (
) )
func (ws *WebhookServer) applyMutatePolicies(request *v1beta1.AdmissionRequest, policyContext *engine.PolicyContext, policies []*v1.ClusterPolicy, ts int64, logger logr.Logger) []byte { func (ws *WebhookServer) applyMutatePolicies(request *v1beta1.AdmissionRequest, policyContext *engine.PolicyContext, policies []*v1.ClusterPolicy, ts int64, logger logr.Logger) []byte {
var triggeredMutatePolicies []v1.ClusterPolicy
var mutateEngineResponses []*response.EngineResponse var mutateEngineResponses []*response.EngineResponse
mutatePatches, triggeredMutatePolicies, mutateEngineResponses := ws.handleMutation(request, policyContext, policies, ts) mutatePatches, mutateEngineResponses := ws.handleMutation(request, policyContext, policies)
logger.V(6).Info("", "generated patches", string(mutatePatches)) logger.V(6).Info("", "generated patches", string(mutatePatches))
admissionReviewLatencyDuration := int64(time.Since(time.Unix(ts, 0))) admissionReviewLatencyDuration := int64(time.Since(time.Unix(ts, 0)))
go registerAdmissionReviewLatencyMetricMutate(logger, *ws.promConfig.Metrics, string(request.Operation), mutateEngineResponses, triggeredMutatePolicies, admissionReviewLatencyDuration, ts) go registerAdmissionReviewDurationMetricMutate(logger, *ws.promConfig.Metrics, string(request.Operation), mutateEngineResponses, admissionReviewLatencyDuration)
go registerAdmissionRequestsMetricMutate(logger, *ws.promConfig.Metrics, string(request.Operation), mutateEngineResponses)
return mutatePatches return mutatePatches
} }
@ -41,11 +41,10 @@ func (ws *WebhookServer) applyMutatePolicies(request *v1beta1.AdmissionRequest,
func (ws *WebhookServer) handleMutation( func (ws *WebhookServer) handleMutation(
request *v1beta1.AdmissionRequest, request *v1beta1.AdmissionRequest,
policyContext *engine.PolicyContext, policyContext *engine.PolicyContext,
policies []*kyverno.ClusterPolicy, policies []*kyverno.ClusterPolicy) ([]byte, []*response.EngineResponse) {
admissionRequestTimestamp int64) ([]byte, []kyverno.ClusterPolicy, []*response.EngineResponse) {
if len(policies) == 0 { if len(policies) == 0 {
return nil, nil, nil return nil, nil
} }
resourceName := request.Kind.Kind + "/" + request.Name resourceName := request.Kind.Kind + "/" + request.Name
@ -60,7 +59,7 @@ func (ws *WebhookServer) handleMutation(
if err != nil { if err != nil {
// as resource cannot be parsed, we skip processing // as resource cannot be parsed, we skip processing
logger.Error(err, "failed to extract resource") logger.Error(err, "failed to extract resource")
return nil, nil, nil return nil, nil
} }
var deletionTimeStamp *metav1.Time var deletionTimeStamp *metav1.Time
if reflect.DeepEqual(newR, unstructured.Unstructured{}) { if reflect.DeepEqual(newR, unstructured.Unstructured{}) {
@ -70,11 +69,10 @@ func (ws *WebhookServer) handleMutation(
} }
if deletionTimeStamp != nil && request.Operation == v1beta1.Update { if deletionTimeStamp != nil && request.Operation == v1beta1.Update {
return nil, nil, nil return nil, nil
} }
var patches [][]byte var patches [][]byte
var engineResponses []*response.EngineResponse var engineResponses []*response.EngineResponse
var triggeredPolicies []kyverno.ClusterPolicy
for _, policy := range policies { for _, policy := range policies {
if !policy.HasMutate() { if !policy.HasMutate() {
@ -99,13 +97,11 @@ func (ws *WebhookServer) handleMutation(
policyContext.NewResource = engineResponse.PatchedResource policyContext.NewResource = engineResponse.PatchedResource
engineResponses = append(engineResponses, engineResponse) engineResponses = append(engineResponses, engineResponse)
// registering the kyverno_policy_rule_results_info metric concurrently // registering the kyverno_policy_results_total metric concurrently
go ws.registerPolicyRuleResultsMetricMutation(logger, string(request.Operation), *policy, *engineResponse, admissionRequestTimestamp) go ws.registerPolicyResultsMetricMutation(logger, string(request.Operation), *policy, *engineResponse)
triggeredPolicies = append(triggeredPolicies, *policy)
// registering the kyverno_policy_rule_execution_latency_milliseconds metric concurrently // registering the kyverno_policy_execution_duration_seconds metric concurrently
go ws.registerPolicyRuleExecutionLatencyMetricMutate(logger, string(request.Operation), *policy, *engineResponse, admissionRequestTimestamp) go ws.registerPolicyExecutionDurationMetricMutate(logger, string(request.Operation), *policy, *engineResponse)
triggeredPolicies = append(triggeredPolicies, *policy)
} }
// generate annotations // generate annotations
@ -137,7 +133,7 @@ func (ws *WebhookServer) handleMutation(
}() }()
// patches holds all the successful patches, if no patch is created, it returns nil // patches holds all the successful patches, if no patch is created, it returns nil
return engineutils.JoinPatches(patches), triggeredPolicies, engineResponses return engineutils.JoinPatches(patches), engineResponses
} }
func (ws *WebhookServer) applyMutation(request *v1beta1.AdmissionRequest, policyContext *engine.PolicyContext, logger logr.Logger) (*response.EngineResponse, [][]byte, error) { func (ws *WebhookServer) applyMutation(request *v1beta1.AdmissionRequest, policyContext *engine.PolicyContext, logger logr.Logger) (*response.EngineResponse, [][]byte, error) {
@ -161,22 +157,22 @@ func (ws *WebhookServer) applyMutation(request *v1beta1.AdmissionRequest, policy
return engineResponse, policyPatches, nil return engineResponse, policyPatches, nil
} }
func (ws *WebhookServer) registerPolicyRuleResultsMetricMutation(logger logr.Logger, resourceRequestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, admissionRequestTimestamp int64) { func (ws *WebhookServer) registerPolicyResultsMetricMutation(logger logr.Logger, resourceRequestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse) {
resourceRequestOperationPromAlias, err := policyRuleResults.ParseResourceRequestOperation(resourceRequestOperation) resourceRequestOperationPromAlias, err := policyResults.ParseResourceRequestOperation(resourceRequestOperation)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_results_info metrics for the above policy", "name", policy.Name) logger.Error(err, "error occurred while registering kyverno_policy_results_total metrics for the above policy", "name", policy.Name)
} }
if err := policyRuleResults.ParsePromMetrics(*ws.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil { if err := policyResults.ParsePromMetrics(*ws.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, resourceRequestOperationPromAlias); err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_results_info metrics for the above policy", "name", policy.Name) logger.Error(err, "error occurred while registering kyverno_policy_results_total metrics for the above policy", "name", policy.Name)
} }
} }
func (ws *WebhookServer) registerPolicyRuleExecutionLatencyMetricMutate(logger logr.Logger, resourceRequestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, admissionRequestTimestamp int64) { func (ws *WebhookServer) registerPolicyExecutionDurationMetricMutate(logger logr.Logger, resourceRequestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse) {
resourceRequestOperationPromAlias, err := policyRuleExecutionLatency.ParseResourceRequestOperation(resourceRequestOperation) resourceRequestOperationPromAlias, err := policyExecutionDuration.ParseResourceRequestOperation(resourceRequestOperation)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_execution_latency_milliseconds metrics for the above policy", "name", policy.Name) logger.Error(err, "error occurred while registering kyverno_policy_execution_duration_seconds metrics for the above policy", "name", policy.Name)
} }
if err := policyRuleExecutionLatency.ParsePromMetrics(*ws.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, "", resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil { if err := policyExecutionDuration.ParsePromMetrics(*ws.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, "", resourceRequestOperationPromAlias); err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_execution_latency_milliseconds metrics for the above policy", "name", policy.Name) logger.Error(err, "error occurred while registering kyverno_policy_execution_duration_seconds metrics for the above policy", "name", policy.Name)
} }
} }

48
pkg/webhooks/server.go
View file

@ -24,7 +24,8 @@ import (
"github.com/kyverno/kyverno/pkg/event" "github.com/kyverno/kyverno/pkg/event"
"github.com/kyverno/kyverno/pkg/generate" "github.com/kyverno/kyverno/pkg/generate"
"github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/metrics"
admissionReviewLatency "github.com/kyverno/kyverno/pkg/metrics/admissionreviewlatency" admissionRequests "github.com/kyverno/kyverno/pkg/metrics/admissionrequests"
admissionReviewDuration "github.com/kyverno/kyverno/pkg/metrics/admissionreviewduration"
"github.com/kyverno/kyverno/pkg/openapi" "github.com/kyverno/kyverno/pkg/openapi"
"github.com/kyverno/kyverno/pkg/policycache" "github.com/kyverno/kyverno/pkg/policycache"
"github.com/kyverno/kyverno/pkg/policyreport" "github.com/kyverno/kyverno/pkg/policyreport"
@ -431,32 +432,53 @@ func failureResponse(message string) *v1beta1.AdmissionResponse {
} }
} }
func registerAdmissionReviewLatencyMetricMutate(logger logr.Logger, promMetrics metrics.PromMetrics, requestOperation string, engineResponses []*response.EngineResponse, triggeredPolicies []v1.ClusterPolicy, admissionReviewLatencyDuration int64, admissionRequestTimestamp int64) { func registerAdmissionReviewDurationMetricMutate(logger logr.Logger, promMetrics metrics.PromMetrics, requestOperation string, engineResponses []*response.EngineResponse, admissionReviewLatencyDuration int64) {
resourceRequestOperationPromAlias, err := admissionReviewLatency.ParseResourceRequestOperation(requestOperation) resourceRequestOperationPromAlias, err := admissionReviewDuration.ParseResourceRequestOperation(requestOperation)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics") logger.Error(err, "error occurred while registering kyverno_admission_review_duration_seconds metrics")
} }
if err := admissionReviewLatency.ParsePromMetrics(promMetrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil { if err := admissionReviewDuration.ParsePromMetrics(promMetrics).ProcessEngineResponses(engineResponses, admissionReviewLatencyDuration, resourceRequestOperationPromAlias); err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics") logger.Error(err, "error occurred while registering kyverno_admission_review_duration_seconds metrics")
} }
} }
func registerAdmissionReviewLatencyMetricGenerate(logger logr.Logger, promMetrics metrics.PromMetrics, requestOperation string, admissionRequestTimestamp int64, latencyReceiver *chan int64, triggeredGeneratePoliciesReceiver *chan []v1.ClusterPolicy, engineResponsesReceiver *chan []*response.EngineResponse) { func registerAdmissionRequestsMetricMutate(logger logr.Logger, promMetrics metrics.PromMetrics, requestOperation string, engineResponses []*response.EngineResponse) {
resourceRequestOperationPromAlias, err := admissionReviewDuration.ParseResourceRequestOperation(requestOperation)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_requests_total metrics")
}
if err := admissionRequests.ParsePromMetrics(promMetrics).ProcessEngineResponses(engineResponses, resourceRequestOperationPromAlias); err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_requests_total metrics")
}
}
func registerAdmissionReviewDurationMetricGenerate(logger logr.Logger, promMetrics metrics.PromMetrics, requestOperation string, latencyReceiver *chan int64, engineResponsesReceiver *chan []*response.EngineResponse) {
defer close(*latencyReceiver) defer close(*latencyReceiver)
defer close(*triggeredGeneratePoliciesReceiver)
defer close(*engineResponsesReceiver) defer close(*engineResponsesReceiver)
triggeredPolicies := <-(*triggeredGeneratePoliciesReceiver)
engineResponses := <-(*engineResponsesReceiver) engineResponses := <-(*engineResponsesReceiver)
resourceRequestOperationPromAlias, err := admissionReviewLatency.ParseResourceRequestOperation(requestOperation) resourceRequestOperationPromAlias, err := admissionReviewDuration.ParseResourceRequestOperation(requestOperation)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics") logger.Error(err, "error occurred while registering kyverno_admission_review_duration_seconds metrics")
} }
// this goroutine will keep on waiting here till it doesn't receive the admission review latency int64 from the other goroutine i.e. ws.HandleGenerate // this goroutine will keep on waiting here till it doesn't receive the admission review latency int64 from the other goroutine i.e. ws.HandleGenerate
admissionReviewLatencyDuration := <-(*latencyReceiver) admissionReviewLatencyDuration := <-(*latencyReceiver)
if err := admissionReviewLatency.ParsePromMetrics(promMetrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil { if err := admissionReviewDuration.ParsePromMetrics(promMetrics).ProcessEngineResponses(engineResponses, admissionReviewLatencyDuration, resourceRequestOperationPromAlias); err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics") logger.Error(err, "error occurred while registering kyverno_admission_review_duration_seconds metrics")
}
}
func registerAdmissionRequestsMetricGenerate(logger logr.Logger, promMetrics metrics.PromMetrics, requestOperation string, engineResponsesReceiver *chan []*response.EngineResponse) {
defer close(*engineResponsesReceiver)
engineResponses := <-(*engineResponsesReceiver)
resourceRequestOperationPromAlias, err := admissionReviewDuration.ParseResourceRequestOperation(requestOperation)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_requests_total metrics")
}
if err := admissionRequests.ParsePromMetrics(promMetrics).ProcessEngineResponses(engineResponses, resourceRequestOperationPromAlias); err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_requests_total metrics")
} }
} }

76
pkg/webhooks/validation.go
View file

@ -1,18 +1,20 @@
package webhooks package webhooks
import ( import (
"github.com/kyverno/kyverno/pkg/event"
"reflect" "reflect"
"time" "time"
"github.com/kyverno/kyverno/pkg/event"
"github.com/go-logr/logr" "github.com/go-logr/logr"
kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1" v1 "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
"github.com/kyverno/kyverno/pkg/engine" "github.com/kyverno/kyverno/pkg/engine"
"github.com/kyverno/kyverno/pkg/engine/response" "github.com/kyverno/kyverno/pkg/engine/response"
"github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/metrics"
admissionReviewLatency "github.com/kyverno/kyverno/pkg/metrics/admissionreviewlatency" admissionRequests "github.com/kyverno/kyverno/pkg/metrics/admissionrequests"
policyRuleExecutionLatency "github.com/kyverno/kyverno/pkg/metrics/policyruleexecutionlatency" admissionReviewDuration "github.com/kyverno/kyverno/pkg/metrics/admissionreviewduration"
policyRuleResults "github.com/kyverno/kyverno/pkg/metrics/policyruleresults" policyExecutionDuration "github.com/kyverno/kyverno/pkg/metrics/policyexecutionduration"
policyResults "github.com/kyverno/kyverno/pkg/metrics/policyresults"
"github.com/kyverno/kyverno/pkg/policyreport" "github.com/kyverno/kyverno/pkg/policyreport"
v1beta1 "k8s.io/api/admission/v1beta1" v1beta1 "k8s.io/api/admission/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -31,7 +33,7 @@ type validationHandler struct {
func (v *validationHandler) handleValidation( func (v *validationHandler) handleValidation(
promConfig *metrics.PromConfig, promConfig *metrics.PromConfig,
request *v1beta1.AdmissionRequest, request *v1beta1.AdmissionRequest,
policies []*kyverno.ClusterPolicy, policies []*v1.ClusterPolicy,
policyContext *engine.PolicyContext, policyContext *engine.PolicyContext,
namespaceLabels map[string]string, namespaceLabels map[string]string,
admissionRequestTimestamp int64) (bool, string) { admissionRequestTimestamp int64) (bool, string) {
@ -55,7 +57,6 @@ func (v *validationHandler) handleValidation(
} }
var engineResponses []*response.EngineResponse var engineResponses []*response.EngineResponse
var triggeredPolicies []kyverno.ClusterPolicy
for _, policy := range policies { for _, policy := range policies {
logger.V(3).Info("evaluating policy", "policy", policy.Name) logger.V(3).Info("evaluating policy", "policy", policy.Name)
policyContext.Policy = *policy policyContext.Policy = *policy
@ -67,13 +68,12 @@ func (v *validationHandler) handleValidation(
continue continue
} }
// registering the kyverno_policy_rule_results_info metric concurrently // registering the kyverno_policy_results_total metric concurrently
go registerPolicyRuleResultsMetricValidation(promConfig, logger, string(request.Operation), policyContext.Policy, *engineResponse, admissionRequestTimestamp) go registerPolicyResultsMetricValidation(promConfig, logger, string(request.Operation), policyContext.Policy, *engineResponse)
// registering the kyverno_policy_rule_execution_latency_milliseconds metric concurrently // registering the kyverno_policy_execution_duration_seconds metric concurrently
go registerPolicyRuleExecutionLatencyMetricValidate(promConfig, logger, string(request.Operation), policyContext.Policy, *engineResponse, admissionRequestTimestamp) go registerPolicyExecutionDurationMetricValidate(promConfig, logger, string(request.Operation), policyContext.Policy, *engineResponse)
engineResponses = append(engineResponses, engineResponse) engineResponses = append(engineResponses, engineResponse)
triggeredPolicies = append(triggeredPolicies, *policy)
if !engineResponse.IsSuccessful() { if !engineResponse.IsSuccessful() {
logger.V(2).Info("validation failed", "policy", policy.Name, "failed rules", engineResponse.GetFailedRules()) logger.V(2).Info("validation failed", "policy", policy.Name, "failed rules", engineResponse.GetFailedRules())
continue continue
@ -102,9 +102,11 @@ func (v *validationHandler) handleValidation(
v.eventGen.Add(events...) v.eventGen.Add(events...)
if blocked { if blocked {
logger.V(4).Info("resource blocked") logger.V(4).Info("resource blocked")
//registering the kyverno_admission_review_latency_milliseconds metric concurrently //registering the kyverno_admission_review_duration_seconds metric concurrently
admissionReviewLatencyDuration := int64(time.Since(time.Unix(admissionRequestTimestamp, 0))) admissionReviewLatencyDuration := int64(time.Since(time.Unix(admissionRequestTimestamp, 0)))
go registerAdmissionReviewLatencyMetricValidate(promConfig, logger, string(request.Operation), engineResponses, triggeredPolicies, admissionReviewLatencyDuration, admissionRequestTimestamp) go registerAdmissionReviewDurationMetricValidate(promConfig, logger, string(request.Operation), engineResponses, admissionReviewLatencyDuration)
//registering the kyverno_admission_requests_total metric concurrently
go registerAdmissionRequestsMetricValidate(promConfig, logger, string(request.Operation), engineResponses)
return false, getEnforceFailureErrorMsg(engineResponses) return false, getEnforceFailureErrorMsg(engineResponses)
} }
@ -116,10 +118,12 @@ func (v *validationHandler) handleValidation(
prInfos := policyreport.GeneratePRsFromEngineResponse(engineResponses, logger) prInfos := policyreport.GeneratePRsFromEngineResponse(engineResponses, logger)
v.prGenerator.Add(prInfos...) v.prGenerator.Add(prInfos...)
//registering the kyverno_admission_review_latency_milliseconds metric concurrently //registering the kyverno_admission_review_duration_seconds metric concurrently
admissionReviewLatencyDuration := int64(time.Since(time.Unix(admissionRequestTimestamp, 0))) admissionReviewLatencyDuration := int64(time.Since(time.Unix(admissionRequestTimestamp, 0)))
go registerAdmissionReviewLatencyMetricValidate(promConfig, logger, string(request.Operation), engineResponses, triggeredPolicies, admissionReviewLatencyDuration, admissionRequestTimestamp) go registerAdmissionReviewDurationMetricValidate(promConfig, logger, string(request.Operation), engineResponses, admissionReviewLatencyDuration)
//registering the kyverno_admission_requests_total metric concurrently
go registerAdmissionRequestsMetricValidate(promConfig, logger, string(request.Operation), engineResponses)
return true, "" return true, ""
} }
@ -132,33 +136,43 @@ func getResourceName(request *v1beta1.AdmissionRequest) string {
return resourceName return resourceName
} }
func registerPolicyRuleResultsMetricValidation(promConfig *metrics.PromConfig, logger logr.Logger, requestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, admissionRequestTimestamp int64) { func registerPolicyResultsMetricValidation(promConfig *metrics.PromConfig, logger logr.Logger, requestOperation string, policy v1.ClusterPolicy, engineResponse response.EngineResponse) {
resourceRequestOperationPromAlias, err := policyRuleResults.ParseResourceRequestOperation(requestOperation) resourceRequestOperationPromAlias, err := policyResults.ParseResourceRequestOperation(requestOperation)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_results_info metrics for the above policy", "name", policy.Name) logger.Error(err, "error occurred while registering kyverno_policy_results_total metrics for the above policy", "name", policy.Name)
} }
if err := policyRuleResults.ParsePromMetrics(*promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil { if err := policyResults.ParsePromMetrics(*promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, resourceRequestOperationPromAlias); err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_results_info metrics for the above policy", "name", policy.Name) logger.Error(err, "error occurred while registering kyverno_policy_results_total metrics for the above policy", "name", policy.Name)
} }
} }
func registerPolicyRuleExecutionLatencyMetricValidate(promConfig *metrics.PromConfig, logger logr.Logger, requestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, admissionRequestTimestamp int64) { func registerPolicyExecutionDurationMetricValidate(promConfig *metrics.PromConfig, logger logr.Logger, requestOperation string, policy v1.ClusterPolicy, engineResponse response.EngineResponse) {
resourceRequestOperationPromAlias, err := policyRuleExecutionLatency.ParseResourceRequestOperation(requestOperation) resourceRequestOperationPromAlias, err := policyExecutionDuration.ParseResourceRequestOperation(requestOperation)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_execution_latency_milliseconds metrics for the above policy", "name", policy.Name) logger.Error(err, "error occurred while registering kyverno_policy_execution_duration_seconds metrics for the above policy", "name", policy.Name)
} }
if err := policyRuleExecutionLatency.ParsePromMetrics(*promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, "", resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil { if err := policyExecutionDuration.ParsePromMetrics(*promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, "", resourceRequestOperationPromAlias); err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_execution_latency_milliseconds metrics for the above policy", "name", policy.Name) logger.Error(err, "error occurred while registering kyverno_policy_execution_duration_seconds metrics for the above policy", "name", policy.Name)
} }
} }
func registerAdmissionReviewLatencyMetricValidate(promConfig *metrics.PromConfig, logger logr.Logger, requestOperation string, engineResponses []*response.EngineResponse, triggeredPolicies []kyverno.ClusterPolicy, admissionReviewLatencyDuration int64, admissionRequestTimestamp int64) { func registerAdmissionReviewDurationMetricValidate(promConfig *metrics.PromConfig, logger logr.Logger, requestOperation string, engineResponses []*response.EngineResponse, admissionReviewLatencyDuration int64) {
resourceRequestOperationPromAlias, err := admissionReviewLatency.ParseResourceRequestOperation(requestOperation) resourceRequestOperationPromAlias, err := admissionReviewDuration.ParseResourceRequestOperation(requestOperation)
if err != nil { if err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics") logger.Error(err, "error occurred while registering kyverno_admission_review_duration_seconds metrics")
} }
if err := admissionReviewLatency.ParsePromMetrics(*promConfig.Metrics).ProcessEngineResponses(engineResponses, triggeredPolicies, admissionReviewLatencyDuration, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil { if err := admissionReviewDuration.ParsePromMetrics(*promConfig.Metrics).ProcessEngineResponses(engineResponses, admissionReviewLatencyDuration, resourceRequestOperationPromAlias); err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_review_latency_milliseconds metrics") logger.Error(err, "error occurred while registering kyverno_admission_review_duration_seconds metrics")
}
}
func registerAdmissionRequestsMetricValidate(promConfig *metrics.PromConfig, logger logr.Logger, requestOperation string, engineResponses []*response.EngineResponse) {
resourceRequestOperationPromAlias, err := admissionRequests.ParseResourceRequestOperation(requestOperation)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_requests_total metrics")
}
if err := admissionRequests.ParsePromMetrics(*promConfig.Metrics).ProcessEngineResponses(engineResponses, resourceRequestOperationPromAlias); err != nil {
logger.Error(err, "error occurred while registering kyverno_admission_requests_total metrics")
} }
} }