feat: added kyverno_policy_rule_results_info metric

Signed-off-by: Yashvardhan Kukreja <yash.kukreja.98@gmail.com>

pkg/engine/generation.go

@@ -14,10 +14,11 @@ import (
 //    - the caller has to check the ruleResponse to determine whether the path exist
 // 2. returns the list of rules that are applicable on this policy and resource, if 1 succeed
 func Generate(policyContext *PolicyContext) (resp *response.EngineResponse) {
-	return filterRules(policyContext)
+	policyStartTime := time.Now()
+	return filterRules(policyContext, policyStartTime)
 }
 
-func filterRules(policyContext *PolicyContext) *response.EngineResponse {
+func filterRules(policyContext *PolicyContext, startTime time.Time) *response.EngineResponse {
 	kind := policyContext.NewResource.GetKind()
 	name := policyContext.NewResource.GetName()
 	namespace := policyContext.NewResource.GetNamespace()
@@ -25,6 +26,9 @@ func filterRules(policyContext *PolicyContext) *response.EngineResponse {
 	resp := &response.EngineResponse{
 		PolicyResponse: response.PolicyResponse{
 			Policy: policyContext.Policy.Name,
+			PolicyStats: response.PolicyStats{
+				PolicyExecutionTimestamp: startTime.Unix(),
+			},
 			Resource: response.ResourceSpec{
 				Kind:       kind,
 				Name:       name,
@@ -76,7 +80,8 @@ func filterRule(rule kyverno.Rule, policyContext *PolicyContext) *response.RuleR
 				Type:    "Generation",
 				Success: false,
 				RuleStats: response.RuleStats{
-					ProcessingTime: time.Since(startTime),
+					ProcessingTime:         time.Since(startTime),
+					RuleExecutionTimestamp: startTime.Unix(),
 				},
 			}
 		}
@@ -111,7 +116,8 @@ func filterRule(rule kyverno.Rule, policyContext *PolicyContext) *response.RuleR
 		Type:    "Generation",
 		Success: true,
 		RuleStats: response.RuleStats{
-			ProcessingTime: time.Since(startTime),
+			ProcessingTime:         time.Since(startTime),
+			RuleExecutionTimestamp: startTime.Unix(),
 		},
 	}
 }

pkg/engine/mutate/overlay.go

@@ -28,6 +28,7 @@ func ProcessOverlay(log logr.Logger, ruleName string, overlay interface{}, resou
 	resp.Type = utils.Mutation.String()
 	defer func() {
 		resp.RuleStats.ProcessingTime = time.Since(startTime)
+		resp.RuleStats.RuleExecutionTimestamp = startTime.Unix()
 		logger.V(4).Info("finished applying overlay rule", "processingTime", resp.RuleStats.ProcessingTime.String())
 	}()
 

pkg/engine/mutate/patchJson6902.go

@@ -21,6 +21,7 @@ func ProcessPatchJSON6902(ruleName string, patchesJSON6902 []byte, resource unst
 	resp.Type = utils.Mutation.String()
 	defer func() {
 		resp.RuleStats.ProcessingTime = time.Since(startTime)
+		resp.RuleStats.RuleExecutionTimestamp = startTime.Unix()
 		logger.V(4).Info("applied JSON6902 patch", "processingTime", resp.RuleStats.ProcessingTime.String())
 	}()
 

pkg/engine/mutate/patches.go

@@ -28,6 +28,7 @@ func ProcessPatches(log logr.Logger, ruleName string, mutation kyverno.Mutation,
 	resp.Type = utils.Mutation.String()
 	defer func() {
 		resp.RuleStats.ProcessingTime = time.Since(startTime)
+		resp.RuleStats.RuleExecutionTimestamp = startTime.Unix()
 		logger.V(4).Info("applied JSON patch", "processingTime", resp.RuleStats.ProcessingTime.String())
 	}()
 

pkg/engine/mutate/strategicMergePatch.go

@@ -26,6 +26,7 @@ func ProcessStrategicMergePatch(ruleName string, overlay interface{}, resource u
 
 	defer func() {
 		resp.RuleStats.ProcessingTime = time.Since(startTime)
+		resp.RuleStats.RuleExecutionTimestamp = startTime.Unix()
 		logger.V(4).Info("finished applying strategicMerge patch", "processingTime", resp.RuleStats.ProcessingTime.String())
 	}()
 

pkg/engine/mutation.go

@@ -147,5 +147,6 @@ func endMutateResultResponse(logger logr.Logger, resp *response.EngineResponse,
 	}
 
 	resp.PolicyResponse.ProcessingTime = time.Since(startTime)
+	resp.PolicyResponse.PolicyExecutionTimestamp = startTime.Unix()
 	logger.V(5).Info("finished processing policy", "processingTime", resp.PolicyResponse.ProcessingTime.String(), "mutationRulesApplied", resp.PolicyResponse.RulesAppliedCount)
 }

pkg/engine/response/response.go

@@ -52,6 +52,8 @@ type PolicyStats struct {
 	ProcessingTime time.Duration `json:"processingTime"`
 	// Count of rules that were applied successfully
 	RulesAppliedCount int `json:"rulesAppliedCount"`
+	// Timestamp of the instant the Policy was triggered
+	PolicyExecutionTimestamp int64 `json:"policyExecutionTimestamp"`
 }
 
 //RuleResponse details for each rule application
@@ -79,6 +81,8 @@ func (rr RuleResponse) ToString() string {
 type RuleStats struct {
 	// time required to apply the rule on the resource
 	ProcessingTime time.Duration `json:"processingTime"`
+	// Timestamp of the instant the rule got triggered
+	RuleExecutionTimestamp int64 `json:"ruleExecutionTimestamp"`
 }
 
 //IsSuccessful checks if any rule has failed or not

pkg/engine/validation.go

@@ -67,6 +67,7 @@ func buildResponse(logger logr.Logger, ctx *PolicyContext, resp *response.Engine
 	resp.PolicyResponse.Resource.APIVersion = resp.PatchedResource.GetAPIVersion()
 	resp.PolicyResponse.ValidationFailureAction = ctx.Policy.Spec.ValidationFailureAction
 	resp.PolicyResponse.ProcessingTime = time.Since(startTime)
+	resp.PolicyResponse.PolicyExecutionTimestamp = startTime.Unix()
 }
 
 func incrementAppliedCount(resp *response.EngineResponse) {
@@ -229,6 +230,7 @@ func validatePatterns(log logr.Logger, ctx context.EvalInterface, resource unstr
 	resp.Type = utils.Validation.String()
 	defer func() {
 		resp.RuleStats.ProcessingTime = time.Since(startTime)
+		resp.RuleStats.RuleExecutionTimestamp = startTime.Unix()
 		logger.V(4).Info("finished processing rule", "processingTime", resp.RuleStats.ProcessingTime.String())
 	}()
 

pkg/metrics/metrics.go

@@ -30,7 +30,7 @@ func NewPromConfig() *PromConfig {
 	}
 	policyRuleResultsMetric := prom.NewGaugeVec(
 		prom.GaugeOpts{
-			Name: "kyverno_policy_rule_results",
+			Name: "kyverno_policy_rule_results_info",
 			Help: "can be used to track the results associated with the policies applied in the user’s cluster, at the level from rule to policy to admission requests.",
 		},
 		policyRuleResultsLabels,

pkg/metrics/policyruleresults/parsers.go

@@ -0,0 +1,39 @@
+package policyruleresults
+
+import (
+	"fmt"
+	"github.com/kyverno/kyverno/pkg/engine/response"
+	"github.com/kyverno/kyverno/pkg/metrics"
+)
+
+func ParsePromMetrics(pm metrics.PromMetrics) PromMetrics {
+	return PromMetrics(pm)
+}
+
+func ParseRuleTypeFromEngineRuleResponse(rule response.RuleResponse) metrics.RuleType {
+	switch rule.Type {
+	case "Validation":
+		return metrics.Validate
+	case "Mutation":
+		return metrics.Mutate
+	case "Generation":
+		return metrics.Generate
+	default:
+		return metrics.EmptyRuleType
+	}
+}
+
+func ParseResourceRequestOperation(requestOperationStr string) (metrics.ResourceRequestOperation, error) {
+	switch requestOperationStr {
+	case "CREATE":
+		return metrics.ResourceCreated, nil
+	case "UPDATE":
+		return metrics.ResourceUpdated, nil
+	case "DELETE":
+		return metrics.ResourceDeleted, nil
+	case "CONNECT":
+		return metrics.ResourceConnected, nil
+	default:
+		return "", fmt.Errorf("Unknown request operation made by resource: %s. Allowed requests: 'CREATE', 'UPDATE', 'DELETE', 'CONNECT'", requestOperationStr)
+	}
+}

pkg/metrics/policyruleresults/policyRuleResults.go

@@ -0,0 +1,107 @@
+package policyruleresults
+
+import (
+	"fmt"
+	kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
+	"github.com/kyverno/kyverno/pkg/engine/response"
+	"github.com/kyverno/kyverno/pkg/metrics"
+	prom "github.com/prometheus/client_golang/prometheus"
+	"time"
+)
+
+func (pm PromMetrics) registerPolicyRuleResultsMetric(
+	policyValidationMode metrics.PolicyValidationMode,
+	policyType metrics.PolicyType,
+	policyBackgroundMode metrics.PolicyBackgroundMode,
+	policyNamespace, policyName string,
+	resourceName, resourceKind, resourceNamespace string,
+	resourceRequestOperation metrics.ResourceRequestOperation,
+	ruleName string,
+	ruleResult metrics.RuleResult,
+	ruleType metrics.RuleType,
+	ruleExecutionCause metrics.RuleExecutionCause,
+	ruleResponse string,
+	mainRequestTriggerTimestamp, policyExecutionTimestamp, ruleExecutionTimestamp int64,
+) error {
+	if policyType == metrics.Cluster {
+		policyNamespace = "-"
+	}
+	pm.PolicyRuleResults.With(prom.Labels{
+		"policy_validation_mode":         string(policyValidationMode),
+		"policy_type":                    string(policyType),
+		"policy_background_mode":         string(policyBackgroundMode),
+		"policy_namespace":               policyNamespace,
+		"policy_name":                    policyName,
+		"resource_name":                  resourceName,
+		"resource_kind":                  resourceKind,
+		"resource_namespace":             resourceNamespace,
+		"resource_request_operation":     string(resourceRequestOperation),
+		"rule_name":                      ruleName,
+		"rule_result":                    string(ruleResult),
+		"rule_type":                      string(ruleType),
+		"rule_execution_cause":           string(ruleExecutionCause),
+		"rule_response":                  ruleResponse,
+		"main_request_trigger_timestamp": fmt.Sprintf("%+v", time.Unix(mainRequestTriggerTimestamp, 0)),
+		"policy_execution_timestamp":     fmt.Sprintf("%+v", time.Unix(policyExecutionTimestamp, 0)),
+		"rule_execution_timestamp":       fmt.Sprintf("%+v", time.Unix(ruleExecutionTimestamp, 0)),
+	}).Set(1)
+	return nil
+}
+
+//policy - policy related data
+//engineResponse - resource and rule related data
+func (pm PromMetrics) ProcessEngineResponse(policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, executionCause metrics.RuleExecutionCause, resourceRequestOperation metrics.ResourceRequestOperation, mainRequestTriggerTimestamp int64) error {
+
+	policyValidationMode, err := metrics.ParsePolicyValidationMode(policy.Spec.ValidationFailureAction)
+	if err != nil {
+		return err
+	}
+	policyType := metrics.Namespaced
+	policyBackgroundMode := metrics.ParsePolicyBackgroundMode(*policy.Spec.Background)
+	policyNamespace := policy.ObjectMeta.Namespace
+	if policyNamespace == "" {
+		policyNamespace = "-"
+		policyType = metrics.Cluster
+	}
+	policyName := policy.ObjectMeta.Name
+
+	policyExecutionTimestamp := engineResponse.PolicyResponse.PolicyExecutionTimestamp
+
+	resourceSpec := engineResponse.PolicyResponse.Resource
+
+	resourceName := resourceSpec.Name
+	resourceKind := resourceSpec.Kind
+	resourceNamespace := resourceSpec.Namespace
+
+	ruleResponses := engineResponse.PolicyResponse.Rules
+
+	for _, rule := range ruleResponses {
+		ruleName := rule.Name
+		ruleType := ParseRuleTypeFromEngineRuleResponse(rule)
+		ruleResponse := rule.Message
+		ruleResult := metrics.Fail
+		if rule.Success {
+			ruleResult = metrics.Pass
+		}
+
+		ruleExecutionTimestamp := rule.RuleStats.RuleExecutionTimestamp
+
+		if err := pm.registerPolicyRuleResultsMetric(
+			policyValidationMode,
+			policyType,
+			policyBackgroundMode,
+			policyNamespace, policyName,
+			resourceName, resourceKind, resourceNamespace,
+			resourceRequestOperation,
+			ruleName,
+			ruleResult,
+			ruleType,
+			executionCause,
+			ruleResponse,
+			mainRequestTriggerTimestamp, policyExecutionTimestamp, ruleExecutionTimestamp,
+		); err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/metrics/policyruleresults/types.go

@@ -0,0 +1,7 @@
+package policyruleresults
+
+import (
+	"github.com/kyverno/kyverno/pkg/metrics"
+)
+
+type PromMetrics metrics.PromMetrics

pkg/policy/existing.go

@@ -11,10 +11,12 @@ import (
 	"github.com/kyverno/kyverno/pkg/common"
 	"github.com/kyverno/kyverno/pkg/engine"
 	"github.com/kyverno/kyverno/pkg/engine/response"
+	"github.com/kyverno/kyverno/pkg/metrics"
+	policyRuleResults "github.com/kyverno/kyverno/pkg/metrics/policyruleresults"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
 
-func (pc *PolicyController) processExistingResources(policy *kyverno.ClusterPolicy) {
+func (pc *PolicyController) processExistingResources(policy *kyverno.ClusterPolicy, backgroundScanTimestamp int64) {
 	logger := pc.log.WithValues("policy", policy.Name)
 	logger.V(4).Info("applying policy to existing resources")
 
@@ -38,8 +40,11 @@ func (pc *PolicyController) processExistingResources(policy *kyverno.ClusterPoli
 				namespaced, _ = pc.rm.GetScope(k)
 			}
 
+			// this tracker would help to ensure that even for multiple namespaces, duplicate metric are not generated
+			metricRegisteredTracker := false
+
 			if !namespaced {
-				pc.applyAndReportPerNamespace(policy, k, "", rule, logger.WithValues("kind", k))
+				pc.applyAndReportPerNamespace(policy, k, "", rule, logger.WithValues("kind", k), backgroundScanTimestamp, &metricRegisteredTracker)
 				continue
 			}
 
@@ -48,7 +53,7 @@ func (pc *PolicyController) processExistingResources(policy *kyverno.ClusterPoli
 				// for kind: Policy, consider only the namespace which the policy belongs to.
 				// for kind: ClusterPolicy, consider all the namespaces.
 				if policy.Namespace == ns || policy.Namespace == "" {
-					pc.applyAndReportPerNamespace(policy, k, ns, rule, logger.WithValues("kind", k).WithValues("ns", ns))
+					pc.applyAndReportPerNamespace(policy, k, ns, rule, logger.WithValues("kind", k).WithValues("ns", ns), backgroundScanTimestamp, &metricRegisteredTracker)
 				}
 			}
 		}
@@ -66,7 +71,7 @@ func (pc *PolicyController) registerResource(gvk string) (err error) {
 	return nil
 }
 
-func (pc *PolicyController) applyAndReportPerNamespace(policy *kyverno.ClusterPolicy, kind string, ns string, rule kyverno.Rule, logger logr.Logger) {
+func (pc *PolicyController) applyAndReportPerNamespace(policy *kyverno.ClusterPolicy, kind string, ns string, rule kyverno.Rule, logger logr.Logger, backgroundScanTimestamp int64, metricAlreadyRegistered *bool) {
 	rMap := pc.getResourcesPerNamespace(kind, ns, rule, logger)
 	excludeAutoGenResources(*policy, rMap, logger)
 	if len(rMap) == 0 {
@@ -79,9 +84,23 @@ func (pc *PolicyController) applyAndReportPerNamespace(policy *kyverno.ClusterPo
 		engineResponses = append(engineResponses, responses...)
 	}
 
+	if !*metricAlreadyRegistered && len(engineResponses) > 0 {
+		for _, engineResponse := range engineResponses {
+			// registering the kyverno_policy_rule_results_info metric concurrently
+			go pc.registerPolicyRuleResultsMetricValidationBS(logger, *policy, *engineResponse, backgroundScanTimestamp)
+		}
+		*metricAlreadyRegistered = true
+	}
+
 	pc.report(policy.Name, engineResponses, logger)
 }
 
+func (pc *PolicyController) registerPolicyRuleResultsMetricValidationBS(logger logr.Logger, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, backgroundScanTimestamp int64) {
+	if err := policyRuleResults.ParsePromMetrics(*pc.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.BackgroundScan, metrics.ResourceCreated, backgroundScanTimestamp); err != nil {
+		logger.Error(err, "error occurred while registering kyverno_policy_rule_results_info metrics for the above policy", "name", policy.Name)
+	}
+}
+
 func (pc *PolicyController) applyPolicy(policy *kyverno.ClusterPolicy, resource unstructured.Unstructured, logger logr.Logger) (engineResponses []*response.EngineResponse) {
 	// pre-processing, check if the policy and resource version has been processed before
 	if !pc.rm.ProcessResource(policy.Name, policy.ResourceVersion, resource.GetKind(), resource.GetNamespace(), resource.GetName(), resource.GetResourceVersion()) {

pkg/policy/validate_controller.go

@@ -624,7 +624,7 @@ func (pc *PolicyController) syncPolicy(key string) error {
 	}
 
 	updateGR(pc.kyvernoClient, policy.Name, grList, logger)
-	pc.processExistingResources(policy)
+	pc.processExistingResources(policy, startTime.Unix())
 	return nil
 }
 

pkg/webhooks/generation.go

@@ -24,6 +24,8 @@ import (
 	"github.com/kyverno/kyverno/pkg/engine/variables"
 	"github.com/kyverno/kyverno/pkg/event"
 	gen "github.com/kyverno/kyverno/pkg/generate"
+	"github.com/kyverno/kyverno/pkg/metrics"
+	policyRuleResults "github.com/kyverno/kyverno/pkg/metrics/policyruleresults"
 	kyvernoutils "github.com/kyverno/kyverno/pkg/utils"
 	"github.com/kyverno/kyverno/pkg/webhooks/generate"
 	v1beta1 "k8s.io/api/admission/v1beta1"
@@ -33,7 +35,7 @@ import (
 )
 
 //HandleGenerate handles admission-requests for policies with generate rules
-func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, policies []*kyverno.ClusterPolicy, ctx *context.Context, userRequestInfo kyverno.RequestInfo, dynamicConfig config.Interface) {
+func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, policies []*kyverno.ClusterPolicy, ctx *context.Context, userRequestInfo kyverno.RequestInfo, dynamicConfig config.Interface, admissionRequestTimestamp int64) {
 	logger := ws.log.WithValues("action", "generation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation, "gvk", request.Kind.String())
 	logger.V(4).Info("incoming request")
 	var engineResponses []*response.EngineResponse
@@ -81,6 +83,8 @@ func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, polic
 					resp: engineResponse,
 				})
 			}
+			// registering the kyverno_policy_rule_results_info metric concurrently
+			go ws.registerPolicyRuleResultsMetricGeneration(logger, string(request.Operation), *policy, *engineResponse, admissionRequestTimestamp)
 		}
 
 		// Adds Generate Request to a channel(queue size 1000) to generators
@@ -98,6 +102,16 @@ func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, polic
 	}
 }
 
+func (ws *WebhookServer) registerPolicyRuleResultsMetricGeneration(logger logr.Logger, resourceRequestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, admissionRequestTimestamp int64) {
+	resourceRequestOperationPromAlias, err := policyRuleResults.ParseResourceRequestOperation(resourceRequestOperation)
+	if err != nil {
+		logger.Error(err, "error occurred while registering kyverno_policy_rule_results_info metrics for the above policy", "name", policy.Name)
+	}
+	if err := policyRuleResults.ParsePromMetrics(*ws.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil {
+		logger.Error(err, "error occurred while registering kyverno_policy_rule_results_info metrics for the above policy", "name", policy.Name)
+	}
+}
+
 //handleUpdate handles admission-requests for update
 func (ws *WebhookServer) handleUpdate(request *v1beta1.AdmissionRequest, policies []*kyverno.ClusterPolicy) {
 	logger := ws.log.WithValues("action", "generation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation, "gvk", request.Kind.String())

pkg/webhooks/mutation.go

@@ -6,6 +6,7 @@ import (
 	"sort"
 	"time"
 
+	"github.com/go-logr/logr"
 	kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
 	v1 "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
 	"github.com/kyverno/kyverno/pkg/common"
@@ -13,6 +14,8 @@ import (
 	"github.com/kyverno/kyverno/pkg/engine/context"
 	"github.com/kyverno/kyverno/pkg/engine/response"
 	engineutils "github.com/kyverno/kyverno/pkg/engine/utils"
+	"github.com/kyverno/kyverno/pkg/metrics"
+	policyRuleResults "github.com/kyverno/kyverno/pkg/metrics/policyruleresults"
 	v1beta1 "k8s.io/api/admission/v1beta1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
@@ -24,7 +27,8 @@ func (ws *WebhookServer) HandleMutation(
 	resource unstructured.Unstructured,
 	policies []*kyverno.ClusterPolicy,
 	ctx *context.Context,
-	userRequestInfo kyverno.RequestInfo) []byte {
+	userRequestInfo kyverno.RequestInfo,
+	admissionRequestTimestamp int64) []byte {
 
 	if len(policies) == 0 {
 		return nil
@@ -87,6 +91,9 @@ func (ws *WebhookServer) HandleMutation(
 
 		policyContext.NewResource = engineResponse.PatchedResource
 		engineResponses = append(engineResponses, engineResponse)
+
+		// registering the kyverno_policy_rule_results_info metric concurrently
+		go ws.registerPolicyRuleResultsMetricMutation(logger, string(request.Operation), *policy, *engineResponse, admissionRequestTimestamp)
 	}
 
 	// generate annotations
@@ -121,6 +128,16 @@ func (ws *WebhookServer) HandleMutation(
 	return engineutils.JoinPatches(patches)
 }
 
+func (ws *WebhookServer) registerPolicyRuleResultsMetricMutation(logger logr.Logger, resourceRequestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, admissionRequestTimestamp int64) {
+	resourceRequestOperationPromAlias, err := policyRuleResults.ParseResourceRequestOperation(resourceRequestOperation)
+	if err != nil {
+		logger.Error(err, "error occurred while registering kyverno_policy_rule_results_info metrics for the above policy", "name", policy.Name)
+	}
+	if err := policyRuleResults.ParsePromMetrics(*ws.promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil {
+		logger.Error(err, "error occurred while registering kyverno_policy_rule_results_info metrics for the above policy", "name", policy.Name)
+	}
+}
+
 type mutateStats struct {
 	resp      *response.EngineResponse
 	namespace string

pkg/webhooks/server.go

@@ -313,6 +313,8 @@ func (ws *WebhookServer) ResourceMutation(request *v1beta1.AdmissionRequest) *v1
 	}
 
 	logger.V(6).Info("received an admission request in mutating webhook")
+	// timestamp at which this admission request got triggered
+	admissionRequestTimestamp := time.Now().Unix()
 	mutatePolicies := ws.pCache.GetPolicyObject(policycache.Mutate, request.Kind.Kind, "")
 	generatePolicies := ws.pCache.GetPolicyObject(policycache.Generate, request.Kind.Kind, "")
 
@@ -360,7 +362,7 @@ func (ws *WebhookServer) ResourceMutation(request *v1beta1.AdmissionRequest) *v1
 	patchedResource := request.Object.Raw
 
 	// MUTATION
-	patches = ws.HandleMutation(request, resource, mutatePolicies, ctx, userRequestInfo)
+	patches = ws.HandleMutation(request, resource, mutatePolicies, ctx, userRequestInfo, admissionRequestTimestamp)
 	logger.V(6).Info("", "generated patches", string(patches))
 
 	// patch the resource with patches before handling validation rules
@@ -370,7 +372,7 @@ func (ws *WebhookServer) ResourceMutation(request *v1beta1.AdmissionRequest) *v1
 	// GENERATE
 	newRequest := request.DeepCopy()
 	newRequest.Object.Raw = patchedResource
-	go ws.HandleGenerate(newRequest, generatePolicies, ctx, userRequestInfo, ws.configHandler)
+	go ws.HandleGenerate(newRequest, generatePolicies, ctx, userRequestInfo, ws.configHandler, admissionRequestTimestamp)
 
 	patchType := v1beta1.PatchTypeJSONPatch
 	return &v1beta1.AdmissionResponse{
@@ -399,6 +401,8 @@ func (ws *WebhookServer) resourceValidation(request *v1beta1.AdmissionRequest) *
 	}
 
 	logger.V(6).Info("received an admission request in validating webhook")
+	// timestamp at which this admission request got triggered
+	admissionRequestTimestamp := time.Now().Unix()
 
 	policies := ws.pCache.GetPolicyObject(policycache.ValidateEnforce, request.Kind.Kind, "")
 	// Get namespace policies from the cache for the requested resource namespace
@@ -450,7 +454,7 @@ func (ws *WebhookServer) resourceValidation(request *v1beta1.AdmissionRequest) *
 		namespaceLabels = common.GetNamespaceSelectorsFromNamespaceLister(request.Kind.Kind, request.Namespace, ws.nsLister, logger)
 	}
 
-	ok, msg := HandleValidation(request, policies, nil, ctx, userRequestInfo, ws.statusListener, ws.eventGen, ws.prGenerator, ws.log, ws.configHandler, ws.resCache, ws.client, namespaceLabels)
+	ok, msg := HandleValidation(ws.promConfig, request, policies, nil, ctx, userRequestInfo, ws.statusListener, ws.eventGen, ws.prGenerator, ws.log, ws.configHandler, ws.resCache, ws.client, namespaceLabels, admissionRequestTimestamp)
 	if !ok {
 		logger.Info("admission request denied")
 		return &v1beta1.AdmissionResponse{

pkg/webhooks/validate_audit.go

@@ -151,7 +151,8 @@ func (h *auditHandler) processNextWorkItem() bool {
 func (h *auditHandler) process(request *v1beta1.AdmissionRequest) error {
 	var roles, clusterRoles []string
 	var err error
-
+	// time at which the corresponding the admission request's processing got initiated
+	admissionRequestTimestamp := time.Now().Unix()
 	logger := h.log.WithName("process")
 	policies := h.pCache.GetPolicyObject(policycache.ValidateAudit, request.Kind.Kind, "")
 	// Get namespace policies from the cache for the requested resource namespace
@@ -180,7 +181,7 @@ func (h *auditHandler) process(request *v1beta1.AdmissionRequest) error {
 		namespaceLabels = common.GetNamespaceSelectorsFromNamespaceLister(request.Kind.Kind, request.Namespace, h.nsLister, logger)
 	}
 
-	HandleValidation(request, policies, nil, ctx, userRequestInfo, h.statusListener, h.eventGen, h.prGenerator, logger, h.configHandler, h.resCache, h.client, namespaceLabels)
+	HandleValidation(h.promConfig, request, policies, nil, ctx, userRequestInfo, h.statusListener, h.eventGen, h.prGenerator, logger, h.configHandler, h.resCache, h.client, namespaceLabels, admissionRequestTimestamp)
 	return nil
 }
 

pkg/webhooks/validation.go

@@ -22,12 +22,16 @@ import (
 	v1beta1 "k8s.io/api/admission/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	"github.com/kyverno/kyverno/pkg/metrics"
+	policyRuleResults "github.com/kyverno/kyverno/pkg/metrics/policyruleresults"
 )
 
 // HandleValidation handles validating webhook admission request
 // If there are no errors in validating rule we apply generation rules
 // patchedResource is the (resource + patches) after applying mutation rules
 func HandleValidation(
+	promConfig *metrics.PromConfig,
 	request *v1beta1.AdmissionRequest,
 	policies []*kyverno.ClusterPolicy,
 	patchedResource []byte,
@@ -40,7 +44,8 @@ func HandleValidation(
 	dynamicConfig config.Interface,
 	resCache resourcecache.ResourceCache,
 	client *client.Client,
-	namespaceLabels map[string]string) (bool, string) {
+	namespaceLabels map[string]string,
+	admissionRequestTimestamp int64) (bool, string) {
 
 	if len(policies) == 0 {
 		return true, ""
@@ -99,6 +104,9 @@ func HandleValidation(
 			continue
 		}
 
+		// registering the kyverno_policy_rule_results_info metric concurrently
+		go registerPolicyRuleResultsMetricValidation(promConfig, logger, string(request.Operation), policyContext.Policy, *engineResponse, admissionRequestTimestamp)
+
 		engineResponses = append(engineResponses, engineResponse)
 		statusListener.Update(validateStats{
 			resp:      engineResponse,
@@ -147,6 +155,16 @@ func HandleValidation(
 	return true, ""
 }
 
+func registerPolicyRuleResultsMetricValidation(promConfig *metrics.PromConfig, logger logr.Logger, requestOperation string, policy kyverno.ClusterPolicy, engineResponse response.EngineResponse, admissionRequestTimestamp int64) {
+	resourceRequestOperationPromAlias, err := policyRuleResults.ParseResourceRequestOperation(requestOperation)
+	if err != nil {
+		logger.Error(err, "error occurred while registering kyverno_policy_rule_results_info metrics for the above policy", "name", policy.Name)
+	}
+	if err := policyRuleResults.ParsePromMetrics(*promConfig.Metrics).ProcessEngineResponse(policy, engineResponse, metrics.AdmissionRequest, resourceRequestOperationPromAlias, admissionRequestTimestamp); err != nil {
+		logger.Error(err, "error occurred while registering kyverno_policy_rule_results_info metrics for the above policy", "name", policy.Name)
+	}
+}
+
 func buildDeletionPrInfo(oldR unstructured.Unstructured) policyreport.Info {
 	return policyreport.Info{
 		Namespace: oldR.GetNamespace(),