feat: use a dedicated policy metrics controller (#4818)

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

cmd/kyverno/main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/kyverno/kyverno/pkg/config"
 	"github.com/kyverno/kyverno/pkg/controllers/certmanager"
 	configcontroller "github.com/kyverno/kyverno/pkg/controllers/config"
+	policymetricscontroller "github.com/kyverno/kyverno/pkg/controllers/metrics/policy"
 	policycachecontroller "github.com/kyverno/kyverno/pkg/controllers/policycache"
 	admissionreportcontroller "github.com/kyverno/kyverno/pkg/controllers/report/admission"
 	aggregatereportcontroller "github.com/kyverno/kyverno/pkg/controllers/report/aggregate"
@@ -589,6 +590,12 @@ func main() {
 		maxQueuedEvents,
 		logging.WithName("EventGenerator"),
 	)
+	// This controller only subscribe to events, nothing is returned...
+	policymetricscontroller.NewController(
+		metricsConfig,
+		kyvernoInformer.Kyverno().V1().ClusterPolicies(),
+		kyvernoInformer.Kyverno().V1().Policies(),
+	)
 	// create non leader controllers
 	nonLeaderControllers, nonLeaderBootstrap := createNonLeaderControllers(
 		kubeInformer,

pkg/controllers/metrics/policy/controller.go

@@ -0,0 +1,81 @@
+package policy
+
+import (
+	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
+	kyvernov1informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1"
+	"github.com/kyverno/kyverno/pkg/metrics"
+	controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
+	kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
+)
+
+type controller struct {
+	// config
+	metricsConfig *metrics.MetricsConfig
+}
+
+// TODO: this is a very strange controller, it only processes events, this should be changed to a real controller
+// but this is difficult as we currently can't remove existing metrics. To be reviewed when we implement a more
+// solid metrics system.
+func NewController(metricsConfig *metrics.MetricsConfig, cpolInformer kyvernov1informers.ClusterPolicyInformer, polInformer kyvernov1informers.PolicyInformer) {
+	c := controller{
+		metricsConfig: metricsConfig,
+	}
+	controllerutils.AddEventHandlers(cpolInformer.Informer(), c.addPolicy, c.updatePolicy, c.deletePolicy)
+	controllerutils.AddEventHandlers(polInformer.Informer(), c.addNsPolicy, c.updateNsPolicy, c.deleteNsPolicy)
+}
+
+func (c *controller) addPolicy(obj interface{}) {
+	p := obj.(*kyvernov1.ClusterPolicy)
+	// register kyverno_policy_rule_info_total metric concurrently
+	go c.registerPolicyRuleInfoMetricAddPolicy(logger, p)
+	// register kyverno_policy_changes_total metric concurrently
+	go c.registerPolicyChangesMetricAddPolicy(logger, p)
+}
+
+func (c *controller) updatePolicy(old, cur interface{}) {
+	oldP, curP := old.(*kyvernov1.ClusterPolicy), cur.(*kyvernov1.ClusterPolicy)
+	// register kyverno_policy_rule_info_total metric concurrently
+	go c.registerPolicyRuleInfoMetricUpdatePolicy(logger, oldP, curP)
+	// register kyverno_policy_changes_total metric concurrently
+	go c.registerPolicyChangesMetricUpdatePolicy(logger, oldP, curP)
+}
+
+func (c *controller) deletePolicy(obj interface{}) {
+	p, ok := kubeutils.GetObjectWithTombstone(obj).(*kyvernov1.ClusterPolicy)
+	if !ok {
+		logger.Info("Failed to get deleted object", "obj", obj)
+		return
+	}
+	// register kyverno_policy_rule_info_total metric concurrently
+	go c.registerPolicyRuleInfoMetricDeletePolicy(logger, p)
+	// register kyverno_policy_changes_total metric concurrently
+	go c.registerPolicyChangesMetricDeletePolicy(logger, p)
+}
+
+func (c *controller) addNsPolicy(obj interface{}) {
+	p := obj.(*kyvernov1.Policy)
+	// register kyverno_policy_rule_info_total metric concurrently
+	go c.registerPolicyRuleInfoMetricAddPolicy(logger, p)
+	// register kyverno_policy_changes_total metric concurrently
+	go c.registerPolicyChangesMetricAddPolicy(logger, p)
+}
+
+func (c *controller) updateNsPolicy(old, cur interface{}) {
+	oldP, curP := old.(*kyvernov1.Policy), cur.(*kyvernov1.Policy)
+	// register kyverno_policy_rule_info_total metric concurrently
+	go c.registerPolicyRuleInfoMetricUpdatePolicy(logger, oldP, curP)
+	// register kyverno_policy_changes_total metric concurrently
+	go c.registerPolicyChangesMetricUpdatePolicy(logger, oldP, curP)
+}
+
+func (c *controller) deleteNsPolicy(obj interface{}) {
+	p, ok := kubeutils.GetObjectWithTombstone(obj).(*kyvernov1.Policy)
+	if !ok {
+		logger.Info("Failed to get deleted object", "obj", obj)
+		return
+	}
+	// register kyverno_policy_rule_info_total metric concurrently
+	go c.registerPolicyRuleInfoMetricDeletePolicy(logger, p)
+	// register kyverno_policy_changes_total metric concurrently
+	go c.registerPolicyChangesMetricDeletePolicy(logger, p)
+}

pkg/controllers/metrics/policy/log.go

@@ -0,0 +1,7 @@
+package policy
+
+import "github.com/kyverno/kyverno/pkg/logging"
+
+const controllerName = "policy-metrics"
+
+var logger = logging.WithName(controllerName)

pkg/controllers/metrics/policy/metrics.go

@@ -9,14 +9,14 @@ import (
 	policyRuleInfoMetric "github.com/kyverno/kyverno/pkg/metrics/policyruleinfo"
 )
 
-func (pc *PolicyController) registerPolicyRuleInfoMetricAddPolicy(logger logr.Logger, p kyvernov1.PolicyInterface) {
+func (pc *controller) registerPolicyRuleInfoMetricAddPolicy(logger logr.Logger, p kyvernov1.PolicyInterface) {
 	err := policyRuleInfoMetric.AddPolicy(pc.metricsConfig, p)
 	if err != nil {
 		logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's creation", "name", p.GetName())
 	}
 }
 
-func (pc *PolicyController) registerPolicyRuleInfoMetricUpdatePolicy(logger logr.Logger, oldP, curP kyvernov1.PolicyInterface) {
+func (pc *controller) registerPolicyRuleInfoMetricUpdatePolicy(logger logr.Logger, oldP, curP kyvernov1.PolicyInterface) {
 	// removing the old rules associated metrics
 	err := policyRuleInfoMetric.RemovePolicy(pc.metricsConfig, oldP)
 	if err != nil {
@@ -29,21 +29,21 @@ func (pc *PolicyController) registerPolicyRuleInfoMetricUpdatePolicy(logger logr
 	}
 }
 
-func (pc *PolicyController) registerPolicyRuleInfoMetricDeletePolicy(logger logr.Logger, p kyvernov1.PolicyInterface) {
+func (pc *controller) registerPolicyRuleInfoMetricDeletePolicy(logger logr.Logger, p kyvernov1.PolicyInterface) {
 	err := policyRuleInfoMetric.RemovePolicy(pc.metricsConfig, p)
 	if err != nil {
 		logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's deletion", "name", p.GetName())
 	}
 }
 
-func (pc *PolicyController) registerPolicyChangesMetricAddPolicy(logger logr.Logger, p kyvernov1.PolicyInterface) {
+func (pc *controller) registerPolicyChangesMetricAddPolicy(logger logr.Logger, p kyvernov1.PolicyInterface) {
 	err := policyChangesMetric.RegisterPolicy(pc.metricsConfig, p, policyChangesMetric.PolicyCreated)
 	if err != nil {
 		logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's creation", "name", p.GetName())
 	}
 }
 
-func (pc *PolicyController) registerPolicyChangesMetricUpdatePolicy(logger logr.Logger, oldP, curP kyvernov1.PolicyInterface) {
+func (pc *controller) registerPolicyChangesMetricUpdatePolicy(logger logr.Logger, oldP, curP kyvernov1.PolicyInterface) {
 	oldSpec := oldP.GetSpec()
 	curSpec := curP.GetSpec()
 	if reflect.DeepEqual(oldSpec, curSpec) {
@@ -62,7 +62,7 @@ func (pc *PolicyController) registerPolicyChangesMetricUpdatePolicy(logger logr.
 	}
 }
 
-func (pc *PolicyController) registerPolicyChangesMetricDeletePolicy(logger logr.Logger, p kyvernov1.PolicyInterface) {
+func (pc *controller) registerPolicyChangesMetricDeletePolicy(logger logr.Logger, p kyvernov1.PolicyInterface) {
 	err := policyChangesMetric.RegisterPolicy(pc.metricsConfig, p, policyChangesMetric.PolicyDeleted)
 	if err != nil {
 		logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's deletion", "name", p.GetName())

pkg/policy/policy_controller.go

@@ -163,11 +163,6 @@ func (pc *PolicyController) addPolicy(obj interface{}) {
 
 	logger.Info("policy created", "uid", p.UID, "kind", "ClusterPolicy", "name", p.Name)
 
-	// register kyverno_policy_rule_info_total metric concurrently
-	go pc.registerPolicyRuleInfoMetricAddPolicy(logger, p)
-	// register kyverno_policy_changes_total metric concurrently
-	go pc.registerPolicyChangesMetricAddPolicy(logger, p)
-
 	if !toggle.AutogenInternals.Enabled() {
 		if p.Spec.Background == nil || p.Spec.ValidationFailureAction == "" || missingAutoGenRules(p, logger) {
 			pol, _ := utilscommon.MutatePolicy(p, logger)
@@ -191,11 +186,6 @@ func (pc *PolicyController) updatePolicy(old, cur interface{}) {
 	oldP := old.(*kyvernov1.ClusterPolicy)
 	curP := cur.(*kyvernov1.ClusterPolicy)
 
-	// register kyverno_policy_rule_info_total metric concurrently
-	go pc.registerPolicyRuleInfoMetricUpdatePolicy(logger, oldP, curP)
-	// register kyverno_policy_changes_total metric concurrently
-	go pc.registerPolicyChangesMetricUpdatePolicy(logger, oldP, curP)
-
 	if !toggle.AutogenInternals.Enabled() {
 		if curP.Spec.Background == nil || curP.Spec.ValidationFailureAction == "" || missingAutoGenRules(curP, logger) {
 			pol, _ := utilscommon.MutatePolicy(curP, logger)
@@ -227,11 +217,6 @@ func (pc *PolicyController) deletePolicy(obj interface{}) {
 		return
 	}
 
-	// register kyverno_policy_rule_info_total metric concurrently
-	go pc.registerPolicyRuleInfoMetricDeletePolicy(logger, p)
-	// register kyverno_policy_changes_total metric concurrently
-	go pc.registerPolicyChangesMetricDeletePolicy(logger, p)
-
 	logger.Info("policy deleted", "uid", p.UID, "kind", "ClusterPolicy", "name", p.Name)
 
 	// do not clean up UR on generate clone (sync=true) policy deletion
@@ -249,11 +234,6 @@ func (pc *PolicyController) addNsPolicy(obj interface{}) {
 	logger := pc.log
 	p := obj.(*kyvernov1.Policy)
 
-	// register kyverno_policy_rule_info_total metric concurrently
-	go pc.registerPolicyRuleInfoMetricAddPolicy(logger, p)
-	// register kyverno_policy_changes_total metric concurrently
-	go pc.registerPolicyChangesMetricAddPolicy(logger, p)
-
 	logger.Info("policy created", "uid", p.UID, "kind", "Policy", "name", p.Name, "namespaces", p.Namespace)
 
 	if !toggle.AutogenInternals.Enabled() {
@@ -279,11 +259,6 @@ func (pc *PolicyController) updateNsPolicy(old, cur interface{}) {
 	oldP := old.(*kyvernov1.Policy)
 	curP := cur.(*kyvernov1.Policy)
 
-	// register kyverno_policy_rule_info_total metric concurrently
-	go pc.registerPolicyRuleInfoMetricUpdatePolicy(logger, oldP, curP)
-	// register kyverno_policy_changes_total metric concurrently
-	go pc.registerPolicyChangesMetricUpdatePolicy(logger, oldP, curP)
-
 	if !toggle.AutogenInternals.Enabled() {
 		if curP.Spec.Background == nil || curP.Spec.ValidationFailureAction == "" || missingAutoGenRules(curP, logger) {
 			nsPol, _ := utilscommon.MutatePolicy(curP, logger)
@@ -315,11 +290,6 @@ func (pc *PolicyController) deleteNsPolicy(obj interface{}) {
 		return
 	}
 
-	// register kyverno_policy_rule_info_total metric concurrently
-	go pc.registerPolicyRuleInfoMetricDeletePolicy(logger, p)
-	// register kyverno_policy_changes_total metric concurrently
-	go pc.registerPolicyChangesMetricDeletePolicy(logger, p)
-
 	logger.Info("policy deleted event", "uid", p.UID, "kind", "Policy", "policy_name", p.Name, "namespaces", p.Namespace)
 
 	pol := p