[Feature] Optional Assertions (#1384)

2024-12-14 11:57:37 +00:00 · 2023-08-17 13:39:14 +03:00 · 2023-08-17 13:39:14 +03:00 · 90aca02ce1
commit 90aca02ce1
parent 2f97f87170
13 changed files with 382 additions and 8 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -3,6 +3,7 @@
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
 - (Maintenance) Bump golang.org/x/net to v0.13.0
 - (Feature) PVCResize action concurrency limit
+- (Feature) Optional Assertions

 ## [1.2.32](https://github.com/arangodb/kube-arangodb/tree/1.2.32) (2023-08-07)
 - (Feature) Backup lifetime - remove Backup once its lifetime has been reached
--- a/22
+++ b/22
@ -46,10 +46,16 @@ ifndef KEEP_GOPATH
 	GOPATH := $(GOBUILDDIR)
 endif

+TEST_BUILD ?= 0
 GOBUILDARGS ?=
 GOBASEVERSION := 1.19
 GOVERSION := $(GOBASEVERSION)-alpine3.17
 DISTRIBUTION := alpine:3.15
+GOBUILDTAGS := $(RELEASE_MODE)
+
+ifeq ($(TEST_BUILD),1)
+GOBUILDTAGS := $(GOBUILDTAGS),test_build
+endif

 PULSAR := $(GOBUILDDIR)/bin/pulsar$(shell go env GOEXE)
 GOASSETSBUILDER := $(GOBUILDDIR)/bin/go-assets-builder$(shell go env GOEXE)
@ -274,16 +280,16 @@ fmt-verify: license-verify

 .PHONY: linter
 linter:
-	@$(GOPATH)/bin/golangci-lint run --build-tags "$(RELEASE_MODE)" $(foreach LINT_EXCLUDE,$(LINT_EXCLUDES),--exclude '$(LINT_EXCLUDE)') ./...
+	@$(GOPATH)/bin/golangci-lint run --build-tags "$(GOBUILDTAGS)" $(foreach LINT_EXCLUDE,$(LINT_EXCLUDES),--exclude '$(LINT_EXCLUDE)') ./...

 .PHONY: linter-fix
 linter-fix:
-	@$(GOPATH)/bin/golangci-lint run --fix --build-tags "$(RELEASE_MODE)" $(foreach LINT_EXCLUDE,$(LINT_EXCLUDES),--exclude '$(LINT_EXCLUDE)') ./...
+	@$(GOPATH)/bin/golangci-lint run --fix --build-tags "$(GOBUILDTAGS)" $(foreach LINT_EXCLUDE,$(LINT_EXCLUDES),--exclude '$(LINT_EXCLUDE)') ./...

 .PHONY: vulncheck
 vulncheck:
 	@echo ">> Checking for known vulnerabilities"
-	@-$(GOPATH)/bin/govulncheck --tags $(RELEASE_MODE) ./...
+	@-$(GOPATH)/bin/govulncheck --tags $(GOBUILDTAGS) ./...

 .PHONY: build
 build: docker manifests
@ -383,13 +389,13 @@ bin-all: $(BIN) $(VBIN_LINUX_AMD64) $(VBIN_LINUX_ARM64)

 $(VBIN_LINUX_AMD64): $(SOURCES) dashboard/assets.go VERSION
 	@mkdir -p $(BINDIR)/$(RELEASE_MODE)/linux/amd64
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${GOBUILDARGS} --tags "$(RELEASE_MODE)" $(COMPILE_DEBUG_FLAGS) -installsuffix netgo -ldflags "-X $(REPOPATH)/pkg/version.version=$(VERSION) -X $(REPOPATH)/pkg/version.buildDate=$(BUILDTIME) -X $(REPOPATH)/pkg/version.build=$(COMMIT)" -o $(VBIN_LINUX_AMD64) ./cmd/main
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${GOBUILDARGS} --tags "$(RELEASE_MODE)" $(COMPILE_DEBUG_FLAGS) -installsuffix netgo -ldflags "-X $(REPOPATH)/pkg/version.version=$(VERSION) -X $(REPOPATH)/pkg/version.buildDate=$(BUILDTIME) -X $(REPOPATH)/pkg/version.build=$(COMMIT)" -o $(VBIN_OPS_LINUX_AMD64) ./cmd/main-ops
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${GOBUILDARGS} --tags "$(GOBUILDTAGS)" $(COMPILE_DEBUG_FLAGS) -installsuffix netgo -ldflags "-X $(REPOPATH)/pkg/version.version=$(VERSION) -X $(REPOPATH)/pkg/version.buildDate=$(BUILDTIME) -X $(REPOPATH)/pkg/version.build=$(COMMIT)" -o $(VBIN_LINUX_AMD64) ./cmd/main
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${GOBUILDARGS} --tags "$(GOBUILDTAGS)" $(COMPILE_DEBUG_FLAGS) -installsuffix netgo -ldflags "-X $(REPOPATH)/pkg/version.version=$(VERSION) -X $(REPOPATH)/pkg/version.buildDate=$(BUILDTIME) -X $(REPOPATH)/pkg/version.build=$(COMMIT)" -o $(VBIN_OPS_LINUX_AMD64) ./cmd/main-ops

 $(VBIN_LINUX_ARM64): $(SOURCES) dashboard/assets.go VERSION
 	@mkdir -p $(BINDIR)/$(RELEASE_MODE)/linux/arm64
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build ${GOBUILDARGS} --tags "$(RELEASE_MODE)" $(COMPILE_DEBUG_FLAGS) -installsuffix netgo -ldflags "-X $(REPOPATH)/pkg/version.version=$(VERSION) -X $(REPOPATH)/pkg/version.buildDate=$(BUILDTIME) -X $(REPOPATH)/pkg/version.build=$(COMMIT)" -o $(VBIN_LINUX_ARM64) ./cmd/main
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build ${GOBUILDARGS} --tags "$(RELEASE_MODE)" $(COMPILE_DEBUG_FLAGS) -installsuffix netgo -ldflags "-X $(REPOPATH)/pkg/version.version=$(VERSION) -X $(REPOPATH)/pkg/version.buildDate=$(BUILDTIME) -X $(REPOPATH)/pkg/version.build=$(COMMIT)" -o $(VBIN_OPS_LINUX_ARM64) ./cmd/main-ops
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build ${GOBUILDARGS} --tags "$(GOBUILDTAGS)" $(COMPILE_DEBUG_FLAGS) -installsuffix netgo -ldflags "-X $(REPOPATH)/pkg/version.version=$(VERSION) -X $(REPOPATH)/pkg/version.buildDate=$(BUILDTIME) -X $(REPOPATH)/pkg/version.build=$(COMMIT)" -o $(VBIN_LINUX_ARM64) ./cmd/main
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build ${GOBUILDARGS} --tags "$(GOBUILDTAGS)" $(COMPILE_DEBUG_FLAGS) -installsuffix netgo -ldflags "-X $(REPOPATH)/pkg/version.version=$(VERSION) -X $(REPOPATH)/pkg/version.buildDate=$(BUILDTIME) -X $(REPOPATH)/pkg/version.build=$(COMMIT)" -o $(VBIN_OPS_LINUX_ARM64) ./cmd/main-ops

 $(BIN): $(VBIN_LINUX_AMD64)
 	@cp "$(VBIN_LINUX_AMD64)" "$(BIN)"
@ -530,7 +536,7 @@ manifests: chart-operator

 .PHONY: run-unit-tests
 run-unit-tests: $(SOURCES)
-	go test --count=1 --tags "$(RELEASE_MODE)" $(TESTVERBOSEOPTIONS) \
+	go test --count=1 --tags "$(GOBUILDTAGS)" $(TESTVERBOSEOPTIONS) \
 		$(REPOPATH)/pkg/apis/backup/... \
 		$(REPOPATH)/pkg/apis/deployment/... \
 		$(REPOPATH)/pkg/apis/replication/... \
--- a/docs/generated/metrics/README.md
+++ b/docs/generated/metrics/README.md
@ -15,6 +15,7 @@
 |                  [arangodb_operator_agency_cache_member_serving](./arangodb_operator_agency_cache_member_serving.md)                  | arangodb_operator |   agency_cache    |  Gauge  | Determines if agency member is reachable                                              |
 |                         [arangodb_operator_agency_cache_present](./arangodb_operator_agency_cache_present.md)                         | arangodb_operator |   agency_cache    |  Gauge  | Determines if local agency cache is present                                           |
 |                         [arangodb_operator_agency_cache_serving](./arangodb_operator_agency_cache_serving.md)                         | arangodb_operator |   agency_cache    |  Gauge  | Determines if agency is serving                                                       |
+|                            [arangodb_operator_engine_assertions](./arangodb_operator_engine_assertions.md)                            | arangodb_operator |      engine       | Counter | Number of assertions invoked during Operator runtime                                  |
 |                            [arangodb_operator_engine_ops_alerts](./arangodb_operator_engine_ops_alerts.md)                            | arangodb_operator |      engine       | Counter | Counter for actions which requires ops attention                                      |
 |                      [arangodb_operator_engine_panics_recovered](./arangodb_operator_engine_panics_recovered.md)                      | arangodb_operator |      engine       | Counter | Number of Panics recovered inside Operator reconciliation loop                        |
 |             [arangodb_operator_kubernetes_client_request_errors](./arangodb_operator_kubernetes_client_request_errors.md)             | arangodb_operator | kubernetes_client | Counter | Number of Kubernetes Client request errors                                            |
--- a/docs/generated/metrics/arangodb_operator_engine_assertions.md
+++ b/docs/generated/metrics/arangodb_operator_engine_assertions.md
@ -0,0 +1,18 @@
+# arangodb_operator_engine_assertions (Counter)
+
+## Description
+
+Number of assertions invoked during Operator runtime
+
+## Labels
+
+| Label | Description   |
+|:-----:|:--------------|
+|  key  | Assertion Key |
+
+
+## Alerting
+
+| Priority |                       Query                        | Description                                 |
+|:--------:|:--------------------------------------------------:|:--------------------------------------------|
+| Warning  | irate(arangodb_operator_engine_assertions[1m]) &gt; 1 | Trigger an alert if OPS attention is needed |
--- a/internal/metrics.yaml
+++ b/internal/metrics.yaml
@ -229,6 +229,17 @@ namespaces:
        labels:
          - key: section
            description: "Panic Section"
+      assertions:
+        shortDescription: "Number of assertions invoked during Operator runtime"
+        description: "Number of assertions invoked during Operator runtime"
+        type: "Counter"
+        labels:
+          - key: key
+            description: "Assertion Key"
+        alertingRules:
+          - priority: Warning
+            query: irate(arangodb_operator_engine_assertions[1m]) > 1
+            description: "Trigger an alert if OPS attention is needed"
      ops_alerts:
        shortDescription: "Counter for actions which requires ops attention"
        description: "Counter for actions which requires ops attention"
--- a/pkg/generated/metric_descriptions/arangodb_operator_engine_assertions.go
+++ b/pkg/generated/metric_descriptions/arangodb_operator_engine_assertions.go
@ -0,0 +1,39 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package metric_descriptions
+
+import "github.com/arangodb/kube-arangodb/pkg/util/metrics"
+
+var (
+	arangodbOperatorEngineAssertions = metrics.NewDescription("arangodb_operator_engine_assertions", "Number of assertions invoked during Operator runtime", []string{`key`}, nil)
+)
+
+func init() {
+	registerDescription(arangodbOperatorEngineAssertions)
+}
+
+func ArangodbOperatorEngineAssertions() metrics.Description {
+	return arangodbOperatorEngineAssertions
+}
+
+func ArangodbOperatorEngineAssertionsCounter(value float64, key string) metrics.Metric {
+	return ArangodbOperatorEngineAssertions().Counter(value, key)
+}
--- a/pkg/util/assertion/assert.go
+++ b/pkg/util/assertion/assert.go
@ -0,0 +1,51 @@
+//
+// DISCLAIMER
+//
+// Copyright 2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package assertion
+
+import (
+	"fmt"
+)
+
+type Key string
+
+const (
+	KeyUnknown Key = ""
+)
+
+func (k Key) Assert(condition bool, msg string, args ...interface{}) {
+	assert(2, condition, k, msg, args...)
+}
+
+func Assert(condition bool, key Key, msg string, args ...interface{}) {
+	assert(2, condition, key, msg, args...)
+}
+
+func assert(skip int, condition bool, key Key, msg string, args ...interface{}) {
+	if !condition {
+		return
+	}
+
+	metricsObject.incKeyMetric(key)
+
+	frames := frames(skip)
+
+	_assert(frames, fmt.Sprintf(msg, args...))
+}
--- a/pkg/util/assertion/assert_exit.go
+++ b/pkg/util/assertion/assert_exit.go
@ -0,0 +1,34 @@
+//
+// DISCLAIMER
+//
+// Copyright 2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+//go:build test_build
+
+package assertion
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"syscall"
+)
+
+func _assert(frames []string, msg string) {
+	log.Str("Frames", fmt.Sprintf("(%s)", strings.Join(frames, " -> "))).Str("Message", msg).Warn("Assertion invoked, exiting")
+	os.Exit(int(syscall.SIGSEGV))
+}
--- a/pkg/util/assertion/assert_log.go
+++ b/pkg/util/assertion/assert_log.go
@ -0,0 +1,31 @@
+//
+// DISCLAIMER
+//
+// Copyright 2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+//go:build !test_build
+
+package assertion
+
+import (
+	"fmt"
+	"strings"
+)
+
+func _assert(frames []string, msg string) {
+	log.Str("Frames", fmt.Sprintf("(%s)", strings.Join(frames, " -> "))).Str("Message", msg).Warn("Assertion invoked")
+}
--- a/pkg/util/assertion/frames.go
+++ b/pkg/util/assertion/frames.go
@ -0,0 +1,54 @@
+//
+// DISCLAIMER
+//
+// Copyright 2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package assertion
+
+import (
+	"fmt"
+	"runtime"
+)
+
+func frames(skip int) []string {
+	callers := make([]uintptr, 32)
+
+	runtime.Callers(skip+2, callers)
+
+	frame := runtime.CallersFrames(callers)
+
+	var frames []runtime.Frame
+
+	for {
+		f, next := frame.Next()
+
+		frames = append(frames, f)
+
+		if !next {
+			break
+		}
+	}
+
+	line := make([]string, len(frames))
+
+	for id := range frames {
+		line[id] = fmt.Sprintf("%s:%d", frames[id].File, frames[id].Line)
+	}
+
+	return line
+}
--- a/pkg/util/assertion/logger.go
+++ b/pkg/util/assertion/logger.go
@ -0,0 +1,27 @@
+//
+// DISCLAIMER
+//
+// Copyright 2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package assertion
+
+import "github.com/arangodb/kube-arangodb/pkg/logging"
+
+var (
+	log = logging.Global().RegisterAndGetLogger("assertion", logging.Warn)
+)
--- a/pkg/util/assertion/metrics.go
+++ b/pkg/util/assertion/metrics.go
@ -0,0 +1,62 @@
+//
+// DISCLAIMER
+//
+// Copyright 2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package assertion
+
+import (
+	"sync"
+
+	"github.com/arangodb/kube-arangodb/pkg/generated/metric_descriptions"
+	"github.com/arangodb/kube-arangodb/pkg/metrics/collector"
+	"github.com/arangodb/kube-arangodb/pkg/util/metrics"
+)
+
+func init() {
+	collector.GetCollector().RegisterMetric(metricsObject)
+}
+
+type metricsObjectType struct {
+	metrics map[Key]int
+	lock    sync.Mutex
+}
+
+var (
+	metricsObject = &metricsObjectType{
+		metrics: map[Key]int{},
+	}
+)
+
+func (m *metricsObjectType) incKeyMetric(key Key) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	m.metrics[key]++
+}
+
+func (m *metricsObjectType) CollectMetrics(in metrics.PushMetric) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	for key, invokes := range m.metrics {
+		in.Push(
+			metric_descriptions.ArangodbOperatorEngineAssertionsCounter(float64(invokes), string(key)),
+		)
+	}
+}
--- a/pkg/util/assertion/metrics_test.go
+++ b/pkg/util/assertion/metrics_test.go
@ -0,0 +1,39 @@
+//
+// DISCLAIMER
+//
+// Copyright 2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package assertion
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_Metrics(t *testing.T) {
+	t.Run("Increase metrics", func(t *testing.T) {
+		require.Equal(t, 0, metricsObject.metrics[KeyUnknown])
+
+		metricsObject.incKeyMetric(KeyUnknown)
+		require.Equal(t, 1, metricsObject.metrics[KeyUnknown])
+
+		metricsObject.incKeyMetric(KeyUnknown)
+		require.Equal(t, 2, metricsObject.metrics[KeyUnknown])
+	})
+}