mirrors/kyverno
1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2024-12-14 11:57:48 +00:00
Code Activity

refactor: kyverno_policy_results metric management (#6781)

Browse source 
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
This commit is contained in:
Charles-Edouard Brétéché 2023-04-04 17:07:43 +02:00 committed by GitHub
parent 2f1ac317f4
commit b2340785fc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 132 additions and 150 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cmd/background-controller/main.go
View file

@ -229,6 +229,7 @@ func main() {
)
engine := engine.NewEngine(
configuration,
metricsConfig.Config(),
dClient,
rclient,
engineapi.DefaultContextLoaderFactory(configMapResolver),

2
cmd/cli/kubectl-kyverno/utils/common/common.go
View file

@ -478,6 +478,7 @@ OuterLoop:
}
eng := engine.NewEngine(
cfg,
config.NewDefaultMetricsConfiguration(),
c.Client,
registryclient.NewOrDie(),
store.ContextLoaderFactory(nil),
@ -1073,6 +1074,7 @@ func initializeMockController(objects []runtime.Object) (*generate.GenerateContr
client.SetDiscovery(dclient.NewFakeDiscoveryClient(nil))
c := generate.NewGenerateControllerWithOnlyClient(client, engine.NewEngine(
config.NewDefaultConfiguration(false),
config.NewDefaultMetricsConfiguration(),
client,
nil,
store.ContextLoaderFactory(nil),

1
cmd/kyverno/main.go
View file

@ -386,6 +386,7 @@ func main() {
}
eng := engine.NewEngine(
configuration,
metricsConfig.Config(),
dClient,
rclient,
engineapi.DefaultContextLoaderFactory(configMapResolver),

1
cmd/reports-controller/main.go
View file

@ -323,6 +323,7 @@ func main() {
}
eng := engine.NewEngine(
configuration,
metricsConfig.Config(),
dClient,
rclient,
engineapi.DefaultContextLoaderFactory(configMapResolver),

57
pkg/engine/engine.go
View file

@ -16,35 +16,52 @@ import (
"github.com/kyverno/kyverno/pkg/engine/internal"
engineutils "github.com/kyverno/kyverno/pkg/engine/utils"
"github.com/kyverno/kyverno/pkg/logging"
"github.com/kyverno/kyverno/pkg/metrics"
"github.com/kyverno/kyverno/pkg/registryclient"
"github.com/kyverno/kyverno/pkg/tracing"
"go.opentelemetry.io/otel/metric/global"
"go.opentelemetry.io/otel/metric/instrument"
"go.opentelemetry.io/otel/trace"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
type engine struct {
configuration config.Configuration
client dclient.Interface
rclient registryclient.Client
contextLoader engineapi.ContextLoaderFactory
exceptionSelector engineapi.PolicyExceptionSelector
configuration config.Configuration
metricsConfiguration config.MetricsConfiguration
client dclient.Interface
rclient registryclient.Client
contextLoader engineapi.ContextLoaderFactory
exceptionSelector engineapi.PolicyExceptionSelector
// metrics
resultCounter instrument.Int64Counter
}
type handlerFactory = func() (handlers.Handler, error)
func NewEngine(
configuration config.Configuration,
metricsConfiguration config.MetricsConfiguration,
client dclient.Interface,
rclient registryclient.Client,
contextLoader engineapi.ContextLoaderFactory,
exceptionSelector engineapi.PolicyExceptionSelector,
) engineapi.Engine {
meter := global.MeterProvider().Meter(metrics.MeterName)
resultCounter, err := meter.Int64Counter(
"kyverno_policy_results",
instrument.WithDescription("can be used to track the results associated with the policies applied in the user’s cluster, at the level from rule to policy to admission requests"),
)
if err != nil {
logging.Error(err, "failed to register metric kyverno_policy_results")
}
return &engine{
configuration: configuration,
client: client,
rclient: rclient,
contextLoader: contextLoader,
exceptionSelector: exceptionSelector,
configuration: configuration,
metricsConfiguration: metricsConfiguration,
client: client,
rclient: rclient,
contextLoader: contextLoader,
exceptionSelector: exceptionSelector,
resultCounter: resultCounter,
}
}
@ -58,7 +75,9 @@ func (e *engine) Validate(
policyResponse := e.validate(ctx, logger, policyContext)
response = response.WithPolicyResponse(policyResponse)
}
return response.Done(time.Now())
response = response.Done(time.Now())
e.reportMetrics(ctx, logger, policyContext.Operation(), policyContext.AdmissionOperation(), response)
return response
}
func (e *engine) Mutate(
@ -73,7 +92,9 @@ func (e *engine) Mutate(
WithPolicyResponse(policyResponse).
WithPatchedResource(patchedResource)
}
return response.Done(time.Now())
response = response.Done(time.Now())
e.reportMetrics(ctx, logger, policyContext.Operation(), policyContext.AdmissionOperation(), response)
return response
}
func (e *engine) Generate(
@ -86,7 +107,9 @@ func (e *engine) Generate(
policyResponse := e.generateResponse(ctx, logger, policyContext)
response = response.WithPolicyResponse(policyResponse)
}
return response.Done(time.Now())
response = response.Done(time.Now())
e.reportMetrics(ctx, logger, policyContext.Operation(), policyContext.AdmissionOperation(), response)
return response
}
func (e *engine) VerifyAndPatchImages(
@ -100,7 +123,9 @@ func (e *engine) VerifyAndPatchImages(
policyResponse, innerIvm := e.verifyAndPatchImages(ctx, logger, policyContext)
response, ivm = response.WithPolicyResponse(policyResponse), innerIvm
}
return response.Done(time.Now()), ivm
response = response.Done(time.Now())
e.reportMetrics(ctx, logger, policyContext.Operation(), policyContext.AdmissionOperation(), response)
return response, ivm
}
func (e *engine) ApplyBackgroundChecks(
@ -113,7 +138,9 @@ func (e *engine) ApplyBackgroundChecks(
policyResponse := e.applyBackgroundChecks(ctx, logger, policyContext)
response = response.WithPolicyResponse(policyResponse)
}
return response.Done(time.Now())
response = response.Done(time.Now())
e.reportMetrics(ctx, logger, policyContext.Operation(), policyContext.AdmissionOperation(), response)
return response
}
func (e *engine) ContextLoader(

6
pkg/engine/image_verify_test.go
View file

@ -161,7 +161,10 @@ var signaturePayloads = [][]byte{
[]byte(`{"critical":{"identity":{"docker-reference":"ghcr.io/kyverno/test-verify-image"},"image":{"docker-manifest-digest":"sha256:b31bfb4d0213f254d361e0079deaaebefa4f82ba7aa76ef82e90b4935ad5b105"},"type":"cosign container image signature"},"optional":null}`),
}
var cfg = config.NewDefaultConfiguration(false)
var (
cfg = config.NewDefaultConfiguration(false)
metricsCfg = config.NewDefaultMetricsConfiguration()
)
func testVerifyAndPatchImages(
ctx context.Context,
@ -172,6 +175,7 @@ func testVerifyAndPatchImages(
) (engineapi.EngineResponse, engineapi.ImageVerificationMetadata) {
e := NewEngine(
cfg,
metricsCfg,
nil,
rclient,
engineapi.DefaultContextLoaderFactory(cmResolver),

74
pkg/engine/metrics.go Normal file
View file

@ -0,0 +1,74 @@
package engine
import (
"context"
"strings"
"github.com/go-logr/logr"
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
engineapi "github.com/kyverno/kyverno/pkg/engine/api"
"github.com/kyverno/kyverno/pkg/metrics"
"go.opentelemetry.io/otel/attribute"
)
func (e *engine) reportMetrics(
ctx context.Context,
logger logr.Logger,
operation kyvernov1.AdmissionOperation,
admissionOperation bool,
response engineapi.EngineResponse,
) {
if e.resultCounter != nil {
policy := response.Policy
if name, namespace, policyType, backgroundMode, validationMode, err := metrics.GetPolicyInfos(policy); err != nil {
logger.Error(err, "failed to get policy infos for metrics reporting")
} else {
if policyType == metrics.Cluster {
namespace = "-"
}
if e.metricsConfiguration.CheckNamespace(namespace) {
resourceSpec := response.Resource
resourceKind := resourceSpec.GetKind()
resourceNamespace := resourceSpec.GetNamespace()
for _, rule := range response.PolicyResponse.Rules {
ruleName := rule.Name
ruleType := metrics.ParseRuleTypeFromEngineRuleResponse(rule)
var ruleResult metrics.RuleResult
switch rule.Status {
case engineapi.RuleStatusPass:
ruleResult = metrics.Pass
case engineapi.RuleStatusFail:
ruleResult = metrics.Fail
case engineapi.RuleStatusWarn:
ruleResult = metrics.Warn
case engineapi.RuleStatusError:
ruleResult = metrics.Error
case engineapi.RuleStatusSkip:
ruleResult = metrics.Skip
default:
ruleResult = metrics.Fail
}
executionCause := metrics.AdmissionRequest
if !admissionOperation {
executionCause = metrics.BackgroundScan
}
commonLabels := []attribute.KeyValue{
attribute.String("policy_validation_mode", string(validationMode)),
attribute.String("policy_type", string(policyType)),
attribute.String("policy_background_mode", string(backgroundMode)),
attribute.String("policy_namespace", namespace),
attribute.String("policy_name", name),
attribute.String("resource_kind", resourceKind),
attribute.String("resource_namespace", resourceNamespace),
attribute.String("resource_request_operation", strings.ToLower(string(operation))),
attribute.String("rule_name", ruleName),
attribute.String("rule_result", string(ruleResult)),
attribute.String("rule_type", string(ruleType)),
attribute.String("rule_execution_cause", string(executionCause)),
}
e.resultCounter.Add(ctx, 1, commonLabels...)
}
}
}
}
}

2
pkg/engine/mutation_test.go
View file

@ -9,6 +9,7 @@ import (
kyverno "github.com/kyverno/kyverno/api/kyverno/v1"
client "github.com/kyverno/kyverno/pkg/clients/dclient"
"github.com/kyverno/kyverno/pkg/config"
engineapi "github.com/kyverno/kyverno/pkg/engine/api"
enginecontext "github.com/kyverno/kyverno/pkg/engine/context"
enginetest "github.com/kyverno/kyverno/pkg/engine/test"
@ -32,6 +33,7 @@ func testMutate(
}
e := NewEngine(
cfg,
config.NewDefaultMetricsConfiguration(),
client,
rclient,
contextLoader,

1
pkg/engine/validation_test.go
View file

@ -31,6 +31,7 @@ func testValidate(
}
e := NewEngine(
cfg,
config.NewDefaultMetricsConfiguration(),
nil,
rclient,
contextLoader,

28
pkg/metrics/metrics.go
View file

@ -30,7 +30,6 @@ const (
type MetricsConfig struct {
// instruments
policyChangesMetric instrument.Int64Counter
policyResultsMetric instrument.Int64Counter
policyExecutionDurationMetric instrument.Float64Histogram
clientQueriesMetric instrument.Int64Counter
@ -41,7 +40,6 @@ type MetricsConfig struct {
type MetricsConfigManager interface {
Config() kconfig.MetricsConfiguration
RecordPolicyResults(ctx context.Context, policyValidationMode PolicyValidationMode, policyType PolicyType, policyBackgroundMode PolicyBackgroundMode, policyNamespace string, policyName string, resourceKind string, resourceNamespace string, resourceRequestOperation ResourceRequestOperation, ruleName string, ruleResult RuleResult, ruleType RuleType, ruleExecutionCause RuleExecutionCause)
RecordPolicyChanges(ctx context.Context, policyValidationMode PolicyValidationMode, policyType PolicyType, policyBackgroundMode PolicyBackgroundMode, policyNamespace string, policyName string, policyChangeType string)
RecordPolicyExecutionDuration(ctx context.Context, policyValidationMode PolicyValidationMode, policyType PolicyType, policyBackgroundMode PolicyBackgroundMode, policyNamespace string, policyName string, ruleName string, ruleResult RuleResult, ruleType RuleType, ruleExecutionCause RuleExecutionCause, ruleExecutionLatency float64)
RecordClientQueries(ctx context.Context, clientQueryOperation ClientQueryOperation, clientType ClientType, resourceKind string, resourceNamespace string)
@ -54,11 +52,6 @@ func (m *MetricsConfig) Config() kconfig.MetricsConfiguration {
func (m *MetricsConfig) initializeMetrics(meterProvider metric.MeterProvider) error {
var err error
meter := meterProvider.Meter(MeterName)
m.policyResultsMetric, err = meter.Int64Counter("kyverno_policy_results", instrument.WithDescription("can be used to track the results associated with the policies applied in the user’s cluster, at the level from rule to policy to admission requests"))
if err != nil {
m.Log.Error(err, "Failed to create instrument, kyverno_policy_results")
return err
}
m.policyChangesMetric, err = meter.Int64Counter("kyverno_policy_changes", instrument.WithDescription("can be used to track all the changes associated with the Kyverno policies present on the cluster such as creation, updates and deletions"))
if err != nil {
m.Log.Error(err, "Failed to create instrument, kyverno_policy_changes")
@ -169,27 +162,6 @@ func NewPrometheusConfig(
return provider, metricsServerMux, nil
}
func (m *MetricsConfig) RecordPolicyResults(ctx context.Context, policyValidationMode PolicyValidationMode, policyType PolicyType, policyBackgroundMode PolicyBackgroundMode, policyNamespace string, policyName string,
resourceKind string, resourceNamespace string, resourceRequestOperation ResourceRequestOperation, ruleName string, ruleResult RuleResult, ruleType RuleType,
ruleExecutionCause RuleExecutionCause,
) {
commonLabels := []attribute.KeyValue{
attribute.String("policy_validation_mode", string(policyValidationMode)),
attribute.String("policy_type", string(policyType)),
attribute.String("policy_background_mode", string(policyBackgroundMode)),
attribute.String("policy_namespace", policyNamespace),
attribute.String("policy_name", policyName),
attribute.String("resource_kind", resourceKind),
attribute.String("resource_namespace", resourceNamespace),
attribute.String("resource_request_operation", string(resourceRequestOperation)),
attribute.String("rule_name", ruleName),
attribute.String("rule_result", string(ruleResult)),
attribute.String("rule_type", string(ruleType)),
attribute.String("rule_execution_cause", string(ruleExecutionCause)),
}
m.policyResultsMetric.Add(ctx, 1, commonLabels...)
}
func (m *MetricsConfig) RecordPolicyChanges(ctx context.Context, policyValidationMode PolicyValidationMode, policyType PolicyType, policyBackgroundMode PolicyBackgroundMode, policyNamespace string, policyName string, policyChangeType string) {
commonLabels := []attribute.KeyValue{
attribute.String("policy_validation_mode", string(policyValidationMode)),

78
pkg/metrics/policyresults/policyResults.go
View file

@ -1,78 +0,0 @@
package policyresults
import (
"context"
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
engineapi "github.com/kyverno/kyverno/pkg/engine/api"
"github.com/kyverno/kyverno/pkg/metrics"
)
func registerPolicyResultsMetric(
ctx context.Context,
m metrics.MetricsConfigManager,
policyValidationMode metrics.PolicyValidationMode,
policyType metrics.PolicyType,
policyBackgroundMode metrics.PolicyBackgroundMode,
policyNamespace, policyName string,
resourceKind, resourceNamespace string,
resourceRequestOperation metrics.ResourceRequestOperation,
ruleName string,
ruleResult metrics.RuleResult,
ruleType metrics.RuleType,
ruleExecutionCause metrics.RuleExecutionCause,
) {
if policyType == metrics.Cluster {
policyNamespace = "-"
}
if m.Config().CheckNamespace(policyNamespace) {
m.RecordPolicyResults(ctx, policyValidationMode, policyType, policyBackgroundMode, policyNamespace, policyName, resourceKind, resourceNamespace, resourceRequestOperation, ruleName, ruleResult, ruleType, ruleExecutionCause)
}
}
// policy - policy related data
// engineResponse - resource and rule related data
func ProcessEngineResponse(ctx context.Context, m metrics.MetricsConfigManager, policy kyvernov1.PolicyInterface, engineResponse engineapi.EngineResponse, executionCause metrics.RuleExecutionCause, resourceRequestOperation metrics.ResourceRequestOperation) error {
name, namespace, policyType, backgroundMode, validationMode, err := metrics.GetPolicyInfos(policy)
if err != nil {
return err
}
resourceSpec := engineResponse.Resource
resourceKind := resourceSpec.GetKind()
resourceNamespace := resourceSpec.GetNamespace()
ruleResponses := engineResponse.PolicyResponse.Rules
for _, rule := range ruleResponses {
ruleName := rule.Name
ruleType := metrics.ParseRuleTypeFromEngineRuleResponse(rule)
var ruleResult metrics.RuleResult
switch rule.Status {
case engineapi.RuleStatusPass:
ruleResult = metrics.Pass
case engineapi.RuleStatusFail:
ruleResult = metrics.Fail
case engineapi.RuleStatusWarn:
ruleResult = metrics.Warn
case engineapi.RuleStatusError:
ruleResult = metrics.Error
case engineapi.RuleStatusSkip:
ruleResult = metrics.Skip
default:
ruleResult = metrics.Fail
}
registerPolicyResultsMetric(
ctx,
m,
validationMode,
policyType,
backgroundMode,
namespace, name,
resourceKind, resourceNamespace,
resourceRequestOperation,
ruleName,
ruleResult,
ruleType,
executionCause,
)
}
return nil
}

1
pkg/webhooks/resource/fake.go
View file

@ -54,6 +54,7 @@ func NewFakeHandlers(ctx context.Context, policyCache policycache.Cache) webhook
pcBuilder: webhookutils.NewPolicyContextBuilder(configuration),
engine: engine.NewEngine(
configuration,
config.NewDefaultMetricsConfiguration(),
dclient,
rclient,
engineapi.DefaultContextLoaderFactory(configMapResolver),

1
pkg/webhooks/resource/generation/handler.go
View file

@ -124,7 +124,6 @@ func (h *generationHandler) handleTrigger(
h.applyGeneration(ctx, request, policy, appliedRules, policyContext)
h.syncTriggerAction(ctx, request, policy, failedRules, policyContext)
go webhookutils.RegisterPolicyResultsMetricGeneration(ctx, h.log, h.metrics, string(request.Operation), policy, engineResponse)
go webhookutils.RegisterPolicyExecutionDurationMetricGenerate(ctx, h.log, h.metrics, string(request.Operation), policy, engineResponse)
}
}

2
pkg/webhooks/resource/mutation/mutation.go
View file

@ -122,8 +122,6 @@ func (v *mutationHandler) applyMutations(
engineResponses = append(engineResponses, *engineResponse)
}
// registering the kyverno_policy_results_total metric concurrently
go webhookutils.RegisterPolicyResultsMetricMutation(context.TODO(), v.log, v.metrics, string(request.Operation), policy, *engineResponse)
// registering the kyverno_policy_execution_duration_seconds metric concurrently
go webhookutils.RegisterPolicyExecutionDurationMetricMutate(context.TODO(), v.log, v.metrics, string(request.Operation), policy, *engineResponse)

2
pkg/webhooks/resource/updaterequest.go
View file

@ -61,8 +61,6 @@ func (h *resourceHandlers) handleMutateExisting(ctx context.Context, logger logr
engineResponses = append(engineResponses, &engineResponse)
}
// registering the kyverno_policy_results_total metric concurrently
go webhookutils.RegisterPolicyResultsMetricMutation(context.TODO(), logger, h.metricsConfig, string(request.Operation), policy, engineResponse)
// registering the kyverno_policy_execution_duration_seconds metric concurrently
go webhookutils.RegisterPolicyExecutionDurationMetricMutate(context.TODO(), logger, h.metricsConfig, string(request.Operation), policy, engineResponse)
}

2
pkg/webhooks/resource/validation/validation.go
View file

@ -110,7 +110,6 @@ func (v *validationHandler) HandleValidation(
return
}
go webhookutils.RegisterPolicyResultsMetricValidation(ctx, logger, v.metrics, string(request.Operation), policyContext.Policy(), engineResponse)
go webhookutils.RegisterPolicyExecutionDurationMetricValidate(ctx, logger, v.metrics, string(request.Operation), policyContext.Policy(), engineResponse)
engineResponses = append(engineResponses, engineResponse)
@ -165,7 +164,6 @@ func (v *validationHandler) buildAuditResponses(
policyContext := policyContext.WithPolicy(policy).WithNamespaceLabels(namespaceLabels)
response := v.engine.Validate(ctx, policyContext)
responses = append(responses, response)
go webhookutils.RegisterPolicyResultsMetricValidation(ctx, v.log, v.metrics, string(request.Operation), policyContext.Policy(), response)
go webhookutils.RegisterPolicyExecutionDurationMetricValidate(ctx, v.log, v.metrics, string(request.Operation), policyContext.Policy(), response)
},
)

2
pkg/webhooks/resource/validation_test.go
View file

@ -1051,6 +1051,7 @@ func TestValidate_failure_action_overrides(t *testing.T) {
eng := engine.NewEngine(
config.NewDefaultConfiguration(false),
config.NewDefaultMetricsConfiguration(),
nil,
registryclient.NewOrDie(),
engineapi.DefaultContextLoaderFactory(nil),
@ -1130,6 +1131,7 @@ func Test_RuleSelector(t *testing.T) {
eng := engine.NewEngine(
config.NewDefaultConfiguration(false),
config.NewDefaultMetricsConfiguration(),
nil,
registryclient.NewOrDie(),
engineapi.DefaultContextLoaderFactory(nil),

21
pkg/webhooks/utils/metrics.go
View file

@ -9,7 +9,6 @@ import (
engineapi "github.com/kyverno/kyverno/pkg/engine/api"
"github.com/kyverno/kyverno/pkg/metrics"
policyExecutionDuration "github.com/kyverno/kyverno/pkg/metrics/policyexecutionduration"
policyResults "github.com/kyverno/kyverno/pkg/metrics/policyresults"
)
type reporterFunc func(metrics.ResourceRequestOperation) error
@ -24,26 +23,6 @@ func registerMetric(logger logr.Logger, m string, requestOperation string, r rep
}
}
// POLICY RESULTS
func RegisterPolicyResultsMetricMutation(ctx context.Context, logger logr.Logger, metricsConfig metrics.MetricsConfigManager, requestOperation string, policy kyvernov1.PolicyInterface, engineResponse engineapi.EngineResponse) {
registerMetric(logger, "kyverno_policy_results_total", requestOperation, func(op metrics.ResourceRequestOperation) error {
return policyResults.ProcessEngineResponse(ctx, metricsConfig, policy, engineResponse, metrics.AdmissionRequest, op)
})
}
func RegisterPolicyResultsMetricValidation(ctx context.Context, logger logr.Logger, metricsConfig metrics.MetricsConfigManager, requestOperation string, policy kyvernov1.PolicyInterface, engineResponse engineapi.EngineResponse) {
registerMetric(logger, "kyverno_policy_results_total", requestOperation, func(op metrics.ResourceRequestOperation) error {
return policyResults.ProcessEngineResponse(ctx, metricsConfig, policy, engineResponse, metrics.AdmissionRequest, op)
})
}
func RegisterPolicyResultsMetricGeneration(ctx context.Context, logger logr.Logger, metricsConfig metrics.MetricsConfigManager, requestOperation string, policy kyvernov1.PolicyInterface, engineResponse engineapi.EngineResponse) {
registerMetric(logger, "kyverno_policy_results_total", requestOperation, func(op metrics.ResourceRequestOperation) error {
return policyResults.ProcessEngineResponse(ctx, metricsConfig, policy, engineResponse, metrics.AdmissionRequest, op)
})
}
// POLICY EXECUTION
func RegisterPolicyExecutionDurationMetricMutate(ctx context.Context, logger logr.Logger, metricsConfig metrics.MetricsConfigManager, requestOperation string, policy kyvernov1.PolicyInterface, engineResponse engineapi.EngineResponse) {