[Documentation] Refactor metrics (Part 1) (#1019)

CHANGELOG.md

@@ -24,6 +24,7 @@
 - (Feature) Change DBServer Cleanup Logic
 - (Feature) Set Logger format
 - (Bugfix) Ensure Wait actions to be present after AddMember
+- (Documentation) Refactor metrics (Part 1)
 
 ## [1.2.13](https://github.com/arangodb/kube-arangodb/tree/1.2.13) (2022-06-07)
 - (Bugfix) Fix arangosync members state inspection

Makefile

@@ -531,3 +531,8 @@ check-community:
 
 _check:
 	@$(MAKE) fmt license-verify linter run-unit-tests bin
+
+generate-documentation: generate-go-documentation fmt
+
+generate-go-documentation:
+	ROOT=$(ROOT) go test --count=1 "$(REPOPATH)/internal/..."

docs/README.md

@@ -4,3 +4,7 @@
 - [Documentation](https://www.arangodb.com/docs/stable/deployment-kubernetes.html)
 - [Design documents](./design/README.md)
 - [Providers](./providers/README.md)
+
+
+# ArangoDB Kubernetes Operator Generated Documentation
+- [ArangoDB Operator Metrics & Alerts](./generated/metrics/README.md)

docs/generated/metrics/README.md

@@ -0,0 +1,9 @@
+# ArangoDB Operator Metrics
+
+## List
+
+|                                   Name                                    |     Namespace     | Group  | Type  | Description                                |
+|:-------------------------------------------------------------------------:|:-----------------:|:------:|:-----:|:-------------------------------------------|
+|  [arangodb_operator_agency_errors](./arangodb_operator_agency_errors.md)  | arangodb_operator | agency | Count | Current count of agency cache fetch errors |
+| [arangodb_operator_agency_fetches](./arangodb_operator_agency_fetches.md) | arangodb_operator | agency | Count | Current count of agency cache fetches      |
+|   [arangodb_operator_agency_index](./arangodb_operator_agency_index.md)   | arangodb_operator | agency | Gauge | Current index of the agency cache          |

docs/generated/metrics/arangodb_operator_agency_errors.md

@@ -0,0 +1,12 @@
+# arangodb_operator_agency_errors (Count)
+
+## Description
+
+Current count of agency cache fetch errors
+
+## Labels
+
+|   Label   | Description          |
+|:---------:|:---------------------|
+| namespace | Deployment Namespace |
+|   name    | Deployment Name      |

docs/generated/metrics/arangodb_operator_agency_fetches.md

@@ -0,0 +1,12 @@
+# arangodb_operator_agency_fetches (Count)
+
+## Description
+
+Current count of agency cache fetches
+
+## Labels
+
+|   Label   | Description          |
+|:---------:|:---------------------|
+| namespace | Deployment Namespace |
+|   name    | Deployment Name      |

docs/generated/metrics/arangodb_operator_agency_index.md

@@ -0,0 +1,12 @@
+# arangodb_operator_agency_index (Gauge)
+
+## Description
+
+Current index of the agency cache
+
+## Labels
+
+|   Label   | Description          |
+|:---------:|:---------------------|
+| namespace | Deployment Namespace |
+|   name    | Deployment Name      |

internal/md/column.go

@@ -0,0 +1,76 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package md
+
+import "k8s.io/apimachinery/pkg/util/uuid"
+
+type ColumnAlign int
+
+const (
+	ColumnRightAlign ColumnAlign = iota
+	ColumnCenterAlign
+	ColumnLeftAlign
+)
+
+type Columns []Column
+
+func (c Columns) Get(id string) (Column, bool) {
+	for _, z := range c {
+		if z.ID() == id {
+			return z, true
+		}
+	}
+
+	return nil, false
+}
+
+type Column interface {
+	Name() string
+	Align() ColumnAlign
+
+	ID() string
+}
+
+func NewColumn(name string, align ColumnAlign) Column {
+	return column{
+		name:  name,
+		id:    string(uuid.NewUUID()),
+		align: align,
+	}
+}
+
+type column struct {
+	name  string
+	id    string
+	align ColumnAlign
+}
+
+func (c column) ID() string {
+	return c.id
+}
+
+func (c column) Name() string {
+	return c.name
+}
+
+func (c column) Align() ColumnAlign {
+	return c.align
+}

internal/md/row.go

@@ -0,0 +1,21 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package md

internal/md/table.go

@@ -0,0 +1,145 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package md
+
+import (
+	"sync"
+
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
+)
+
+func NewTable(columns ...Column) Table {
+	return &table{
+		columns: columns,
+	}
+}
+
+type Table interface {
+	Render() string
+
+	AddRow(in map[Column]string) error
+}
+
+type table struct {
+	lock sync.Mutex
+
+	columns Columns
+	rows    []map[string]string
+}
+
+func (t *table) AddRow(in map[Column]string) error {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	r := map[string]string{}
+
+	for k, v := range in {
+		if _, ok := t.columns.Get(k.ID()); !ok {
+			return errors.Newf("Column not found")
+		}
+
+		r[k.ID()] = v
+	}
+
+	t.rows = append(t.rows, r)
+
+	return nil
+}
+
+func (t *table) fillString(base, filler string, align ColumnAlign, size int) string {
+	for len(base) < size {
+		switch align {
+		case ColumnLeftAlign:
+			base += filler
+		case ColumnRightAlign:
+			base = filler + base
+		case ColumnCenterAlign:
+			base += filler
+			if len(base) < size {
+				base = filler + base
+			}
+		}
+	}
+
+	return base
+}
+
+func (t *table) Render() string {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	ks := map[string]int{}
+
+	for _, c := range t.columns {
+		ks[c.ID()] = len(c.Name())
+	}
+
+	for _, r := range t.rows {
+		for _, c := range t.columns {
+			if q := len(r[c.ID()]); q > ks[c.ID()] {
+				ks[c.ID()] = q
+			}
+		}
+	}
+
+	buff := ""
+
+	buff += "|"
+
+	for _, c := range t.columns {
+		buff += " "
+		buff += t.fillString(c.Name(), " ", c.Align(), ks[c.ID()])
+		buff += " |"
+	}
+	buff += "\n|"
+
+	for _, c := range t.columns {
+		switch c.Align() {
+		case ColumnLeftAlign, ColumnCenterAlign:
+			buff += ":"
+		default:
+			buff += "-"
+		}
+
+		buff += t.fillString("", "-", ColumnLeftAlign, ks[c.ID()])
+		switch c.Align() {
+		case ColumnRightAlign, ColumnCenterAlign:
+			buff += ":"
+		default:
+			buff += "-"
+		}
+		buff += "|"
+	}
+	buff += "\n"
+
+	for _, r := range t.rows {
+		buff += "|"
+
+		for _, c := range t.columns {
+			buff += " "
+			buff += t.fillString(r[c.ID()], " ", c.Align(), ks[c.ID()])
+			buff += " |"
+		}
+		buff += "\n"
+	}
+
+	return buff
+}

internal/metrics.go

@@ -0,0 +1,362 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package internal
+
+import (
+	_ "embed"
+	"fmt"
+	"html/template"
+	"os"
+	"path"
+	"sort"
+	"strings"
+
+	"github.com/arangodb/kube-arangodb/internal/md"
+)
+
+//go:embed metrics.go.tmpl
+var metricsGoTemplate []byte
+
+//go:embed metrics.item.go.tmpl
+var metricsItemGoTemplate []byte
+
+//go:embed metrics.tmpl
+var metricsTemplate []byte
+
+//go:embed metrics.item.tmpl
+var metricItemTemplate []byte
+
+//go:embed metrics.yaml
+var metricsData []byte
+
+type MetricsDoc struct {
+	Destination   string `json:"destination" yaml:"destination"`
+	Documentation string `json:"documentation" yaml:"documentation"`
+
+	Namespaces Namespaces `json:"namespaces" yaml:"namespaces"`
+}
+
+type Namespaces map[string]Groups
+
+func (n Namespaces) Keys() []string {
+	r := make([]string, 0, len(n))
+
+	for k := range n {
+		r = append(r, k)
+	}
+
+	sort.Strings(r)
+
+	return r
+}
+
+type Groups map[string]Metrics
+
+func (n Groups) Keys() []string {
+	r := make([]string, 0, len(n))
+
+	for k := range n {
+		r = append(r, k)
+	}
+
+	sort.Strings(r)
+
+	return r
+}
+
+type Metrics map[string]Metric
+
+func (n Metrics) Keys() []string {
+	r := make([]string, 0, len(n))
+
+	for k := range n {
+		r = append(r, k)
+	}
+
+	sort.Strings(r)
+
+	return r
+}
+
+type Metric struct {
+	Description      string `json:"description" yaml:"description"`
+	Type             string `json:"type" yaml:"type"`
+	ShortDescription string `json:"shortDescription" yaml:"shortDescription"`
+
+	Labels        []Label    `json:"labels" yaml:"labels"`
+	AlertingRules []Alerting `json:"alertingRules" yaml:"alertingRules"`
+}
+
+type Alerting struct {
+	Priority    string `json:"priority" yaml:"priority"`
+	Query       string `json:"query" yaml:"query"`
+	Description string `json:"description" yaml:"description"`
+}
+
+type Label struct {
+	Key         string `json:"key" yaml:"key"`
+	Description string `json:"description" yaml:"description"`
+}
+
+func GenerateMetricsDocumentation(root string, in MetricsDoc) error {
+	docsRoot := path.Join(root, in.Documentation)
+	goFilesRoot := path.Join(root, in.Destination)
+
+	if _, err := os.Stat(docsRoot); err != nil {
+		if os.IsNotExist(err) {
+			if err := os.MkdirAll(docsRoot, 0755); err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+	}
+
+	if _, err := os.Stat(goFilesRoot); err != nil {
+		if os.IsNotExist(err) {
+			if err := os.MkdirAll(goFilesRoot, 0755); err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+	}
+
+	if err := generateMetricsREADME(docsRoot, in); err != nil {
+		return err
+	}
+
+	if err := generateMetricsGO(goFilesRoot, in); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func generateMetricFile(root, name string, m Metric) error {
+	key := md.NewColumn("Label", md.ColumnCenterAlign)
+	description := md.NewColumn("Description", md.ColumnLeftAlign)
+	priority := md.NewColumn("Priority", md.ColumnCenterAlign)
+	query := md.NewColumn("Query", md.ColumnCenterAlign)
+	t := md.NewTable(
+		key,
+		description,
+	)
+
+	for _, l := range m.Labels {
+		if err := t.AddRow(map[md.Column]string{
+			key:         l.Key,
+			description: l.Description,
+		}); err != nil {
+			return err
+		}
+	}
+
+	ta := md.NewTable(
+		priority,
+		query,
+		description,
+	)
+
+	for _, l := range m.AlertingRules {
+		if err := ta.AddRow(map[md.Column]string{
+			priority:    l.Priority,
+			query:       l.Query,
+			description: l.Description,
+		}); err != nil {
+			return err
+		}
+	}
+
+	q, err := template.New("metrics").Parse(string(metricItemTemplate))
+	if err != nil {
+		return err
+	}
+
+	out, err := os.OpenFile(path.Join(root, fmt.Sprintf("%s.md", name)), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+
+	if err := q.Execute(out, map[string]interface{}{
+		"name":           name,
+		"type":           m.Type,
+		"description":    m.Description,
+		"labels_table":   t.Render(),
+		"labels":         len(m.Labels) > 0,
+		"alerting_table": ta.Render(),
+		"alerting":       len(m.AlertingRules) > 0,
+	}); err != nil {
+		return err
+	}
+
+	if err := out.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func generateMetricsREADME(root string, in MetricsDoc) error {
+	name := md.NewColumn("Name", md.ColumnCenterAlign)
+	ns := md.NewColumn("Namespace", md.ColumnCenterAlign)
+	group := md.NewColumn("Group", md.ColumnCenterAlign)
+	typeCol := md.NewColumn("Type", md.ColumnCenterAlign)
+	description := md.NewColumn("Description", md.ColumnLeftAlign)
+	t := md.NewTable(
+		name,
+		ns,
+		group,
+		typeCol,
+		description,
+	)
+
+	for _, namespace := range in.Namespaces.Keys() {
+		for _, g := range in.Namespaces[namespace].Keys() {
+			for _, metric := range in.Namespaces[namespace][g].Keys() {
+				mname := fmt.Sprintf("%s_%s_%s", namespace, g, metric)
+				rname := fmt.Sprintf("[%s](./%s.md)", mname, mname)
+
+				details := in.Namespaces[namespace][g][metric]
+
+				if err := t.AddRow(map[md.Column]string{
+					name:        rname,
+					ns:          namespace,
+					group:       g,
+					description: details.ShortDescription,
+					typeCol:     details.Type,
+				}); err != nil {
+					return err
+				}
+
+				if err := generateMetricFile(root, mname, details); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	table := t.Render()
+
+	q, err := template.New("metrics").Parse(string(metricsTemplate))
+	if err != nil {
+		return err
+	}
+
+	out, err := os.OpenFile(path.Join(root, "README.md"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+
+	if err := q.Execute(out, map[string]interface{}{
+		"table": table,
+	}); err != nil {
+		return err
+	}
+
+	if err := out.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func generateLabels(labels []Label) string {
+	if len(labels) == 0 {
+		return "nil"
+	}
+
+	parts := make([]string, len(labels))
+
+	for id := range labels {
+		parts[id] = fmt.Sprintf("`%s`", labels[id].Key)
+	}
+
+	return fmt.Sprintf("[]string{%s}", strings.Join(parts, ", "))
+}
+
+func generateMetricsGO(root string, in MetricsDoc) error {
+	i, err := template.New("metrics").Parse(string(metricsItemGoTemplate))
+
+	if err != nil {
+		return err
+	}
+	for _, namespace := range in.Namespaces.Keys() {
+		for _, g := range in.Namespaces[namespace].Keys() {
+			for _, metric := range in.Namespaces[namespace][g].Keys() {
+				details := in.Namespaces[namespace][g][metric]
+
+				mname := fmt.Sprintf("%s_%s_%s", namespace, g, metric)
+
+				out, err := os.OpenFile(path.Join(root, fmt.Sprintf("%s.go", mname)), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+				if err != nil {
+					return err
+				}
+
+				parts := strings.Split(mname, "_")
+				tparts := strings.Split(strings.Title(strings.Join(parts, " ")), " ")
+
+				fnameParts := make([]string, len(parts))
+				for id := range parts {
+					if id == 0 {
+						fnameParts[id] = parts[id]
+					} else {
+						fnameParts[id] = tparts[id]
+					}
+				}
+
+				if err := i.Execute(out, map[string]interface{}{
+					"name":             mname,
+					"fname":            strings.Join(fnameParts, ""),
+					"ename":            strings.Join(tparts, ""),
+					"shortDescription": details.ShortDescription,
+					"labels":           generateLabels(details.Labels),
+				}); err != nil {
+					return err
+				}
+
+				if err := out.Close(); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	out, err := os.OpenFile(path.Join(root, "metrics.go"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+
+	q, err := template.New("metrics").Parse(string(metricsGoTemplate))
+	if err != nil {
+		return err
+	}
+
+	if err := q.Execute(out, nil); err != nil {
+		return err
+	}
+
+	if err := out.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}

internal/metrics.go.tmpl

@@ -18,12 +18,32 @@
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
 
-package deployment
+package metric_descriptions
 
-import "github.com/arangodb/kube-arangodb/pkg/metrics"
+import (
+	"sync"
+	"github.com/arangodb/kube-arangodb/pkg/util/metrics"
+)
 
 var (
-	inspectDeploymentAgencyIndex   = metrics.MustRegisterGaugeVec(metricsComponent, "inspect_deployment_agency_index", "Index of the agency cache", metrics.DeploymentName)
+	descriptions []metrics.Description
-	inspectDeploymentAgencyFetches = metrics.MustRegisterCounterVec(metricsComponent, "inspect_deployment_agency_fetches", "Number of agency fetches", metrics.DeploymentName)
+	descriptionsLock sync.Mutex
-	inspectDeploymentAgencyErrors  = metrics.MustRegisterCounterVec(metricsComponent, "inspect_deployment_agency_errors", "Number of agency errors", metrics.DeploymentName)
 )
+
+func registerDescription( d ... metrics.Description) {
+	if len(d) == 0 {
+		return
+	}
+
+	descriptionsLock.Lock()
+	defer descriptionsLock.Unlock()
+
+	descriptions = append(descriptions, d...)
+}
+
+func Descriptions (c metrics.PushDescription) {
+	descriptionsLock.Lock()
+	defer descriptionsLock.Unlock()
+
+	c.Push(descriptions...)
+}

internal/metrics.item.go.tmpl

@@ -0,0 +1,35 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package metric_descriptions
+
+import "github.com/arangodb/kube-arangodb/pkg/util/metrics"
+
+var (
+    {{ .fname }} = metrics.NewDescription("{{ .name }}", "{{ .shortDescription }}", {{ .labels }}, nil)
+)
+
+func init() {
+    registerDescription({{ .fname }})
+}
+
+func {{ .ename }}() metrics.Description {
+    return {{ .fname }}
+}

internal/metrics.item.tmpl

@@ -0,0 +1,19 @@
+# {{ .name }} ({{ .type }})
+
+## Description
+
+{{ .description }}
+
+{{ if .labels -}}
+## Labels
+
+{{ .labels_table }}
+
+{{- end -}}
+{{- if .alerting }}
+
+## Alerting
+
+{{ .alerting_table }}
+
+{{- end -}}

internal/metrics.tmpl

@@ -0,0 +1,5 @@
+# ArangoDB Operator Metrics
+
+## List
+
+{{ .table }}

internal/metrics.yaml

@@ -0,0 +1,35 @@
+---
+  
+documentation: docs/generated/metrics
+destination: pkg/generated/metric_descriptions
+
+namespaces:
+  arangodb_operator:
+    agency:
+      index:
+        shortDescription: "Current index of the agency cache"
+        description: "Current index of the agency cache"
+        type: "Gauge"
+        labels:
+          - key: namespace
+            description: "Deployment Namespace"
+          - key: name
+            description: "Deployment Name"
+      fetches:
+        shortDescription: "Current count of agency cache fetches"
+        description: "Current count of agency cache fetches"
+        type: "Count"
+        labels:
+          - key: namespace
+            description: "Deployment Namespace"
+          - key: name
+            description: "Deployment Name"
+      errors:
+        shortDescription: "Current count of agency cache fetch errors"
+        description: "Current count of agency cache fetch errors"
+        type: "Count"
+        labels:
+          - key: namespace
+            description: "Deployment Namespace"
+          - key: name
+            description: "Deployment Name"

internal/metrics_test.go

@@ -0,0 +1,40 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package internal
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func Test_GenerateMetricsDocumentation(t *testing.T) {
+	root := os.Getenv("ROOT")
+	require.NotEmpty(t, root)
+
+	var m MetricsDoc
+
+	require.NoError(t, yaml.Unmarshal(metricsData, &m))
+
+	require.NoError(t, GenerateMetricsDocumentation(root, m))
+}

pkg/deployment/deployment.go

@@ -136,6 +136,14 @@ type Deployment struct {
 	haveServiceMonitorCRD     bool
 
 	memberState memberState.StateInspector
+
+	metrics struct {
+		agency struct {
+			errors  uint64
+			fetches uint64
+			index   uint64
+		}
+	}
 }
 
 func (d *Deployment) WithArangoMember(cache inspectorInterface.Inspector, timeout time.Duration, name string) reconciler.ArangoMemberModContext {

pkg/deployment/deployment_inspector.go

@@ -254,12 +254,12 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva
 		nextInterval = interval
 	}
 
-	inspectDeploymentAgencyFetches.WithLabelValues(d.GetName()).Inc()
+	d.metrics.agency.fetches++
 	if offset, err := d.RefreshAgencyCache(ctx); err != nil {
-		inspectDeploymentAgencyErrors.WithLabelValues(d.GetName()).Inc()
+		d.metrics.agency.errors++
 		d.log.Err(err).Error("Unable to refresh agency")
 	} else {
-		inspectDeploymentAgencyIndex.WithLabelValues(d.GetName()).Set(float64(offset))
+		d.metrics.agency.index = offset
 	}
 
 	// Refresh maintenance lock

pkg/deployment/metrics.go

@@ -24,6 +24,7 @@ import (
 	"sync"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/generated/metric_descriptions"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/throttle"
 	"github.com/arangodb/kube-arangodb/pkg/util/metrics"
 	"github.com/prometheus/client_golang/prometheus"
@@ -66,7 +67,10 @@ func (i *inventory) Describe(descs chan<- *prometheus.Desc) {
 	i.lock.Lock()
 	defer i.lock.Unlock()
 
-	metrics.NewPushDescription(descs).Push(i.deploymentsMetric, i.deploymentMetricsMembersMetric, i.deploymentAgencyStateMetric, i.deploymentShardLeadersMetric, i.deploymentShardsMetric, i.operatorStateRefreshMetric)
+	pd := metrics.NewPushDescription(descs)
+	pd.Push(i.deploymentsMetric, i.deploymentMetricsMembersMetric, i.deploymentAgencyStateMetric, i.deploymentShardLeadersMetric, i.deploymentShardsMetric, i.operatorStateRefreshMetric)
+
+	pd.Push(metric_descriptions.ArangodbOperatorAgencyErrors(), metric_descriptions.ArangodbOperatorAgencyFetches(), metric_descriptions.ArangodbOperatorAgencyIndex())
 }
 
 func (i *inventory) Collect(m chan<- prometheus.Metric) {
@@ -78,6 +82,8 @@ func (i *inventory) Collect(m chan<- prometheus.Metric) {
 		for _, deployment := range deployments {
 			p.Push(i.deploymentsMetric.Gauge(1, deployment.GetNamespace(), deployment.GetName()))
 
+			deployment.CollectMetrics(p)
+
 			if state := deployment.acs.CurrentClusterCache(); state != nil {
 				t := state.GetThrottles()
 
@@ -149,3 +155,9 @@ func (i *inventory) Add(d *Deployment) {
 
 	i.deployments[namespace][name] = d
 }
+
+func (d *Deployment) CollectMetrics(m metrics.PushMetric) {
+	m.Push(metric_descriptions.ArangodbOperatorAgencyErrors().Gauge(float64(d.metrics.agency.errors), d.namespace, d.name))
+	m.Push(metric_descriptions.ArangodbOperatorAgencyFetches().Gauge(float64(d.metrics.agency.fetches), d.namespace, d.name))
+	m.Push(metric_descriptions.ArangodbOperatorAgencyIndex().Gauge(float64(d.metrics.agency.index), d.namespace, d.name))
+}

pkg/generated/metric_descriptions/arangodb_operator_agency_errors.go

@@ -0,0 +1,35 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package metric_descriptions
+
+import "github.com/arangodb/kube-arangodb/pkg/util/metrics"
+
+var (
+	arangodbOperatorAgencyErrors = metrics.NewDescription("arangodb_operator_agency_errors", "Current count of agency cache fetch errors", []string{`namespace`, `name`}, nil)
+)
+
+func init() {
+	registerDescription(arangodbOperatorAgencyErrors)
+}
+
+func ArangodbOperatorAgencyErrors() metrics.Description {
+	return arangodbOperatorAgencyErrors
+}

pkg/generated/metric_descriptions/arangodb_operator_agency_fetches.go

@@ -0,0 +1,35 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package metric_descriptions
+
+import "github.com/arangodb/kube-arangodb/pkg/util/metrics"
+
+var (
+	arangodbOperatorAgencyFetches = metrics.NewDescription("arangodb_operator_agency_fetches", "Current count of agency cache fetches", []string{`namespace`, `name`}, nil)
+)
+
+func init() {
+	registerDescription(arangodbOperatorAgencyFetches)
+}
+
+func ArangodbOperatorAgencyFetches() metrics.Description {
+	return arangodbOperatorAgencyFetches
+}

pkg/generated/metric_descriptions/arangodb_operator_agency_index.go

@@ -0,0 +1,35 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package metric_descriptions
+
+import "github.com/arangodb/kube-arangodb/pkg/util/metrics"
+
+var (
+	arangodbOperatorAgencyIndex = metrics.NewDescription("arangodb_operator_agency_index", "Current index of the agency cache", []string{`namespace`, `name`}, nil)
+)
+
+func init() {
+	registerDescription(arangodbOperatorAgencyIndex)
+}
+
+func ArangodbOperatorAgencyIndex() metrics.Description {
+	return arangodbOperatorAgencyIndex
+}

pkg/generated/metric_descriptions/metrics.go

@@ -0,0 +1,50 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package metric_descriptions
+
+import (
+	"sync"
+
+	"github.com/arangodb/kube-arangodb/pkg/util/metrics"
+)
+
+var (
+	descriptions     []metrics.Description
+	descriptionsLock sync.Mutex
+)
+
+func registerDescription(d ...metrics.Description) {
+	if len(d) == 0 {
+		return
+	}
+
+	descriptionsLock.Lock()
+	defer descriptionsLock.Unlock()
+
+	descriptions = append(descriptions, d...)
+}
+
+func Descriptions(c metrics.PushDescription) {
+	descriptionsLock.Lock()
+	defer descriptionsLock.Unlock()
+
+	c.Push(descriptions...)
+}