Merge pull request #1288 from marquiz/devel/metrics

Improve metrics

pkg/nfd-master/metrics.go

@@ -29,8 +29,8 @@ import (
 // When adding metric names, see https://prometheus.io/docs/practices/naming/#metric-names
 const (
 	buildInfoQuery         = "nfd_master_build_info"
-	updatedNodesQuery      = "nfd_updated_nodes"
-	crdProcessingTimeQuery = "nfd_crd_processing_time"
+	nodeUpdatesQuery       = "nfd_node_updates_total"
+	nfrProcessingTimeQuery = "nfd_nodefeaturerule_processing_duration_seconds"
 )
 
 var (
@@ -43,14 +43,21 @@ var (
 			"version": version.Get(),
 		},
 	})
-	updatedNodes = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: updatedNodesQuery,
+	nodeUpdates = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: nodeUpdatesQuery,
 		Help: "Number of nodes updated by the master.",
 	})
-	crdProcessingTime = prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: crdProcessingTimeQuery,
-		Help: "Time spent processing the NodeFeatureRule CRD.",
-	})
+	nfrProcessingTime = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    nfrProcessingTimeQuery,
+			Help:    "Time processing time of NodeFeatureRule objects.",
+			Buckets: []float64{0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01},
+		},
+		[]string{
+			"name",
+			"node",
+		},
+	)
 )
 
 // registerVersion exposes the Operator build version.
@@ -62,8 +69,8 @@ func registerVersion(version string) {
 func runMetricsServer(port int) {
 	r := prometheus.NewRegistry()
 	r.MustRegister(buildInfo,
-		updatedNodes,
-		crdProcessingTime)
+		nodeUpdates,
+		nfrProcessingTime)
 
 	mux := http.NewServeMux()
 	mux.Handle("/metrics", promhttp.HandlerFor(r, promhttp.HandlerOpts{}))

pkg/nfd-master/nfd-master.go

@@ -854,7 +854,6 @@ func (m *nfdMaster) refreshNodeFeatures(cli *kubernetes.Clientset, nodeName stri
 		return err
 	}
 
-	updatedNodes.Inc()
 	return nil
 }
 
@@ -980,6 +979,7 @@ func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha
 	// Process all rule CRs
 	processStart := time.Now()
 	for _, spec := range ruleSpecs {
+		t := time.Now()
 		switch {
 		case klog.V(3).Enabled():
 			klog.InfoS("executing NodeFeatureRule", "nodefeaturerule", klog.KObj(spec), "nodeName", nodeName, "nodeFeatureRuleSpec", utils.DelayedDumper(spec.Spec))
@@ -1004,9 +1004,9 @@ func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha
 			features.InsertAttributeFeatures(nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Labels)
 			features.InsertAttributeFeatures(nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Vars)
 		}
+		nfrProcessingTime.WithLabelValues(spec.Name, nodeName).Observe(time.Since(t).Seconds())
 	}
 	processingTime := time.Since(processStart)
-	crdProcessingTime.Set(float64(processingTime))
 	klog.V(2).InfoS("processed NodeFeatureRule objects", "nodeName", nodeName, "objectCount", len(ruleSpecs), "duration", processingTime)
 
 	return labels, extendedResources, taints
@@ -1072,6 +1072,7 @@ func (m *nfdMaster) updateNodeObject(cli *kubernetes.Clientset, nodeName string,
 	}
 
 	if len(patches) > 0 || len(statusPatches) > 0 {
+		nodeUpdates.Inc()
 		klog.InfoS("node updated", "nodeName", nodeName)
 	} else {
 		klog.V(1).InfoS("no updates to node", "nodeName", nodeName)

pkg/nfd-worker/metrics.go

@@ -38,10 +38,11 @@ var (
 
 	featureDiscoveryDuration = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Name: featureDiscoveryDurationQuery,
-			Help: "Time taken to discover features",
+			Name:    featureDiscoveryDurationQuery,
+			Help:    "Time taken to discover features",
+			Buckets: []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1},
 		},
-		[]string{"NodeName"},
+		[]string{"node"},
 	)
 	buildInfo = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: buildInfoQuery,