[Bugfix] Prevent LifeCycle restarts (#1127)

CHANGELOG.md

@@ -5,6 +5,7 @@
 - (Feature) Add Member Update helpers
 - (Feature) Active Member condition
 - (Bugfix) Accept Initial Spec
+- (Bugfix) Prevent LifeCycle restarts
 
 ## [1.2.17](https://github.com/arangodb/kube-arangodb/tree/1.2.17) (2022-09-22)
 - (Feature) Add new field to DeploymentReplicationStatus with details on DC2DC sync status=

go.mod

@@ -55,9 +55,6 @@ require (
 	k8s.io/apimachinery v0.21.10
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/klog v1.0.0
-)
-
-require (
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
@@ -106,6 +103,10 @@ require (
 	k8s.io/utils v0.0.0-20210521133846-da695404a2bc // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 	sigs.k8s.io/yaml v1.2.0
+	github.com/josephburnett/jd v1.6.1
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/swag v0.21.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 )
-
-require github.com/cenkalti/backoff/v4 v4.1.3 // indirect

go.sum

@@ -164,12 +164,16 @@ github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
 github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
 github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU=
+github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
@@ -305,6 +309,10 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/jessevdk/go-assets v0.0.0-20160921144138-4f4301a06e15 h1:cW/amwGEJK5MSKntPXRjX4dxs/nGxGT8gXKIsKFmHGc=
 github.com/jessevdk/go-assets v0.0.0-20160921144138-4f4301a06e15/go.mod h1:Fdm/oWRW+CH8PRbLntksCNtmcCBximKPkVQYvmMl80k=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/josephburnett/jd v1.6.1 h1:Uzqhcje4WqvVyp85F3Oj0ezISPTlnhnr/KaLZIy8qh0=
+github.com/josephburnett/jd v1.6.1/go.mod h1:R8ZnZnLt2D4rhW4NvBc/USTo6mzyNT6fYNIIWOJA9GY=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -339,6 +347,9 @@ github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPK
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -900,6 +911,7 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=

pkg/deployment/resources/internal_exporter.go

@@ -19,14 +19,10 @@
 package resources
 
 import (
-	"os"
-	"path/filepath"
-
 	core "k8s.io/api/core/v1"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/apis/shared"
-	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes"
 )
@@ -36,11 +32,7 @@ func ArangodbInternalExporterContainer(image string, args []string, livenessProb
 	resources core.ResourceRequirements, securityContext *core.SecurityContext,
 	spec api.DeploymentSpec) (core.Container, error) {
 
-	binaryPath, err := os.Executable()
+	exePath := k8sutil.LifecycleBinary()
-	if err != nil {
-		return core.Container{}, errors.WithStack(err)
-	}
-	exePath := filepath.Join(k8sutil.LifecycleVolumeMountDir, filepath.Base(binaryPath))
 
 	c := core.Container{
 		Name:    shared.ExporterContainerName,

pkg/deployment/resources/pod_creator_probes.go

@@ -22,8 +22,6 @@ package resources
 
 import (
 	"math"
-	"os"
-	"path/filepath"
 	"time"
 
 	core "k8s.io/api/core/v1"
@@ -223,18 +221,13 @@ func (r *Resources) probeBuilders(imageInfo api.ImageInfo) map[api.ServerGroup]p
 	}
 }
 
-func (r *Resources) probeCommand(spec api.DeploymentSpec, probeType api.ProbeType) ([]string, error) {
+func (r *Resources) probeCommand(spec api.DeploymentSpec, probeType api.ProbeType) []string {
-	binaryPath, err := os.Executable()
+	exePath := k8sutil.LifecycleBinary()
-	if err != nil {
-		return nil, err
-	}
-	exePath := filepath.Join(k8sutil.LifecycleVolumeMountDir, filepath.Base(binaryPath))
 	args := []string{
 		exePath,
 		"lifecycle",
 		"probe",
 		string(probeType),
-		// TODO test rotation required when changed
 	}
 
 	if spec.IsSecure() {
@@ -245,7 +238,7 @@ func (r *Resources) probeCommand(spec api.DeploymentSpec, probeType api.ProbeTyp
 		args = append(args, "--auth")
 	}
 
-	return args, nil
+	return args
 }
 
 func (r *Resources) probeBuilderLivenessCoreSelect(group api.ServerGroup, imageInfo api.ImageInfo) probeBuilder {
@@ -266,10 +259,7 @@ func (r *Resources) probeBuilderStartupCoreSelect(group api.ServerGroup, imageIn
 
 func (r *Resources) probeBuilderLivenessCoreOperator(spec api.DeploymentSpec, group api.ServerGroup,
 	image api.ImageInfo) (Probe, error) {
-	args, err := r.probeCommand(spec, api.ProbeTypeLiveness)
+	args := r.probeCommand(spec, api.ProbeTypeLiveness)
-	if err != nil {
-		return nil, err
-	}
 
 	cmdProbeConfig := &probes.CMDProbeConfig{
 		Command: args,
@@ -283,10 +273,7 @@ func (r *Resources) probeBuilderLivenessCoreOperator(spec api.DeploymentSpec, gr
 
 func (r *Resources) probeBuilderStartupCoreOperator(spec api.DeploymentSpec, group api.ServerGroup,
 	image api.ImageInfo) (Probe, error) {
-	args, err := r.probeCommand(spec, api.ProbeTypeStartUp)
+	args := r.probeCommand(spec, api.ProbeTypeStartUp)
-	if err != nil {
-		return nil, err
-	}
 
 	retries, periodSeconds := getProbeRetries(group)
 	if IsServerProgressAvailable(group, image) {
@@ -399,10 +386,7 @@ func (r *Resources) probeBuilderReadinessCoreSelect() probeBuilder {
 }
 
 func (r *Resources) probeBuilderReadinessCoreOperator(spec api.DeploymentSpec, _ api.ServerGroup, _ api.ImageInfo) (Probe, error) {
-	args, err := r.probeCommand(spec, api.ProbeTypeReadiness)
+	args := r.probeCommand(spec, api.ProbeTypeReadiness)
-	if err != nil {
-		return nil, err
-	}
 
 	return &probes.CMDProbeConfig{
 		Command:             args,

pkg/deployment/rotation/arangod_containers.go

@@ -32,6 +32,7 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/topology"
 	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 )
 
 const (
@@ -61,7 +62,7 @@ func containersCompare(ds api.DeploymentSpec, g api.ServerGroup, spec, status *c
 
 					g := podContainerFuncGenerator(ds, g, ac, bc)
 
-					if m, p, err := comparePodContainer(builder, g(compareServerContainerVolumeMounts)); err != nil {
+					if m, p, err := comparePodContainer(builder, g(compareServerContainerVolumeMounts), g(compareServerContainerProbes)); err != nil {
 						log.Err(err).Msg("Error while getting pod diff")
 						return SkippedRotation, nil, err
 					} else {
@@ -104,11 +105,6 @@ func containersCompare(ds api.DeploymentSpec, g api.ServerGroup, spec, status *c
 						bc.Ports = ac.Ports
 						mode = mode.And(SilentRotation)
 					}
-
-					if !areProbesEqual(ac.StartupProbe, bc.StartupProbe) {
-						bc.StartupProbe = ac.StartupProbe
-						mode = mode.And(SilentRotation)
-					}
 				} else {
 					if ac.Image != bc.Image {
 						// Image changed
@@ -252,6 +248,18 @@ func areProbesEqual(a, b *core.Probe) bool {
 	return equality.Semantic.DeepEqual(a, b)
 }
 
+func isManagedProbe(a, b *core.Probe) bool {
+	if a.Exec == nil || b.Exec == nil {
+		return false
+	}
+
+	if len(a.Exec.Command) == 0 || len(b.Exec.Command) == 0 {
+		return false
+	}
+
+	return a.Exec.Command[0] == k8sutil.LifecycleBinary() && b.Exec.Command[0] == k8sutil.LifecycleBinary()
+}
+
 // areEnvsFromEqual returns true when environment variables from source are the same after filtering.
 func areEnvsFromEqual(a, b []core.EnvFromSource, rules ...func(a, b map[string]core.EnvFromSource) (map[string]core.EnvFromSource, map[string]core.EnvFromSource)) bool {
 	am := createEnvsFromMap(a)

pkg/deployment/rotation/arangod_probes.go

@@ -0,0 +1,65 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package rotation
+
+import (
+	core "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+)
+
+func compareServerContainerProbes(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.Container) comparePodContainerFunc {
+	return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
+		if !areProbesEqual(spec.StartupProbe, status.StartupProbe) {
+			status.StartupProbe = spec.StartupProbe
+			mode = mode.And(SilentRotation)
+		}
+
+		if !areProbesEqual(spec.ReadinessProbe, status.ReadinessProbe) {
+			if isManagedProbe(spec.ReadinessProbe, status.ReadinessProbe) {
+				q := status.ReadinessProbe.DeepCopy()
+
+				q.Exec = spec.ReadinessProbe.Exec.DeepCopy()
+
+				if equality.Semantic.DeepDerivative(spec.ReadinessProbe, q) {
+					status.ReadinessProbe = spec.ReadinessProbe
+					mode = mode.And(SilentRotation)
+				}
+			}
+		}
+
+		if !areProbesEqual(spec.LivenessProbe, status.LivenessProbe) {
+			if isManagedProbe(spec.LivenessProbe, status.LivenessProbe) {
+				q := status.LivenessProbe.DeepCopy()
+
+				q.Exec = spec.LivenessProbe.Exec.DeepCopy()
+
+				if equality.Semantic.DeepDerivative(spec.LivenessProbe, q) {
+					status.LivenessProbe = spec.LivenessProbe
+					mode = mode.And(SilentRotation)
+				}
+			}
+		}
+
+		return
+	}
+}

pkg/deployment/rotation/compare.go

@@ -23,6 +23,7 @@ package rotation
 import (
 	"encoding/json"
 
+	jd "github.com/josephburnett/jd/lib"
 	"github.com/rs/zerolog/log"
 	core "k8s.io/api/core/v1"
 
@@ -114,14 +115,27 @@ func compare(deploymentSpec api.DeploymentSpec, member api.MemberStatus, group a
 	}
 
 	if spec.RotationNeeded(newStatus) {
-		specData, _ := json.Marshal(spec)
+		line := logger.Str("id", member.ID)
-		statusData, _ := json.Marshal(newStatus)
 
-		log.Info().Str("before", spec.PodSpecChecksum).
+		specBytes, errA := json.Marshal(spec.PodSpec)
-			Str("id", member.ID).
+		if errA == nil {
-			Str("spec", string(specData)).
+			line = line.Str("spec", string(specBytes))
-			Str("status", string(statusData)).
+		}
-			Msg("Pod needs rotation - templates does not match")
+
+		statusBytes, errB := json.Marshal(newStatus.PodSpec)
+		if errB == nil {
+			line = line.Str("status", string(statusBytes))
+		}
+
+		if errA == nil && errB == nil {
+			if specData, err := jd.ReadJsonString(string(specBytes)); err == nil && specData != nil {
+				if statusData, err := jd.ReadJsonString(string(statusBytes)); err == nil && statusData != nil {
+					line = line.Str("diff", specData.Diff(statusData).Render())
+				}
+			}
+		}
+
+		line.Info("Pod needs rotation - templates does not match")
 
 		return GracefulRotation, nil, nil
 	}

pkg/deployment/rotation/logger.go

@@ -0,0 +1,27 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package rotation
+
+import "github.com/arangodb/kube-arangodb/pkg/logging"
+
+var (
+	logger = logging.Global().RegisterAndGetLogger("pod_compare", logging.Info)
+)

pkg/deployment/rotation/predefined_test.go

@@ -0,0 +1,76 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package rotation
+
+import (
+	_ "embed"
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	core "k8s.io/api/core/v1"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+)
+
+func init() {
+	k8sutil.SetBinaryPath("arangodb_operator")
+}
+
+//go:embed testdata/pod_lifecycle_change.000.spec.json
+var podLifecycleChange000Spec []byte
+
+//go:embed testdata/pod_lifecycle_change.000.status.json
+var podLifecycleChange000Status []byte
+
+func runPredefinedTests(t *testing.T, spec, status []byte) (mode Mode, plan api.Plan, err error) {
+	var specO, statusO core.PodTemplateSpec
+
+	require.NoError(t, json.Unmarshal(spec, &specO))
+	require.NoError(t, json.Unmarshal(status, &statusO))
+
+	specC, err := resources.ChecksumArangoPod(api.ServerGroupSpec{}, resources.CreatePodFromTemplate(&specO))
+	require.NoError(t, err)
+
+	statusC, err := resources.ChecksumArangoPod(api.ServerGroupSpec{}, resources.CreatePodFromTemplate(&statusO))
+	require.NoError(t, err)
+
+	obj := api.DeploymentSpec{}
+	member := api.MemberStatus{}
+
+	specT, err := api.GetArangoMemberPodTemplate(&specO, specC)
+	require.NoError(t, err)
+	statusT, err := api.GetArangoMemberPodTemplate(&statusO, statusC)
+	require.NoError(t, err)
+
+	return compare(obj, member, api.ServerGroupUnknown, specT, statusT)
+}
+
+func Test_PredefinedTests(t *testing.T) {
+	t.Run("podLifecycleChange000", func(t *testing.T) {
+		mode, plan, err := runPredefinedTests(t, podLifecycleChange000Spec, podLifecycleChange000Status)
+		require.NoError(t, err)
+		require.Empty(t, plan)
+		require.Equal(t, SilentRotation, mode)
+	})
+}

pkg/deployment/rotation/testdata/pod_lifecycle_change.000.spec.json

@@ -0,0 +1,351 @@
+{
+  "metadata": {
+    "creationTimestamp": null,
+    "finalizers": [
+      "pod.database.arangodb.com/delay",
+      "database.arangodb.com/graceful-shutdown"
+    ],
+    "labels": {
+      "app": "arangodb",
+      "arango_deployment": "deployment",
+      "deployment.arangodb.com/member": "CRDN-0000000",
+      "role": "coordinator"
+    },
+    "name": "deployment-crdn-0000000-49237a"
+  },
+  "spec": {
+    "containers": [
+      {
+        "command": [
+          "/usr/sbin/arangod",
+          "--cluster.agency-endpoint=ssl://10.0.111.156:8529",
+          "--cluster.agency-endpoint=ssl://10.0.142.55:8529",
+          "--cluster.agency-endpoint=ssl://10.0.208.123:8529",
+          "--cluster.my-address=ssl://10.0.238.70:8529",
+          "--cluster.my-role=COORDINATOR",
+          "--database.directory=/data",
+          "--foxx.queues=true",
+          "--log.level=INFO",
+          "--log.output=+",
+          "--rocksdb.encryption-keyfile=/secrets/rocksdb/encryption/key",
+          "--server.authentication=true",
+          "--server.endpoint=ssl://[::]:8529",
+          "--server.jwt-secret-folder=/secrets/cluster/jwt",
+          "--server.statistics=true",
+          "--server.storage-engine=rocksdb",
+          "--ssl.ecdh-curve=",
+          "--ssl.keyfile=/secrets/tls/tls.keyfile",
+          "--log.thread=true",
+          "--log.thread-name=true",
+          "--server.authentication-system-only=false",
+          "--javascript.startup-options-denylist=.*",
+          "--javascript.endpoints-allowlist=example.com",
+          "--javascript.environment-variables-allowlist=^HOSTNAME$",
+          "--javascript.environment-variables-allowlist=^PATH$",
+          "--javascript.files-allowlist=^$",
+          "--javascript.harden",
+          "--server.harden",
+          "--javascript.files-allowlist=^(?!(\\/bin\/.*))(?!(\/dev\/.*))(?!(\/etc\/.*))(?!(\/lib\/.*))(?!(\/lifecycle\/.*))(?!(\/media\/.*))(?!(\/mnt\/.*))(?!(\/opt\/.*))(?!(\/proc\/.*))(?!(\/root\/.*))(?!(\/run\/.*))(?!(\/sbin\/.*))(?!(\/secrets\/.*))(?!(\/srv\/.*))(?!(\/usr\/.*))(?!(\/var\/.*)).+",
+          "--backup.api-enabled=jwt",
+          "--cluster.max-number-of-shards=1",
+          "--cluster.min-replication-factor=3",
+          "--cluster.max-replication-factor=3",
+          "--cluster.upgrade=online",
+          "--server.export-metrics-api=true",
+          "--cluster.force-one-shard=true"
+        ],
+        "env": [
+          {
+            "name": "MY_POD_NAME",
+            "valueFrom": {
+              "fieldRef": {
+                "fieldPath": "metadata.name"
+              }
+            }
+          },
+          {
+            "name": "MY_POD_NAMESPACE",
+            "valueFrom": {
+              "fieldRef": {
+                "fieldPath": "metadata.namespace"
+              }
+            }
+          },
+          {
+            "name": "MY_NODE_NAME",
+            "valueFrom": {
+              "fieldRef": {
+                "fieldPath": "spec.nodeName"
+              }
+            }
+          },
+          {
+            "name": "NODE_NAME",
+            "valueFrom": {
+              "fieldRef": {
+                "fieldPath": "spec.nodeName"
+              }
+            }
+          },
+          {
+            "name": "ARANGODB_OVERRIDE_DETECTED_NUMBER_OF_CORES",
+            "value": "1"
+          },
+          {
+            "name": "ARANGODB_OVERRIDE_DETECTED_TOTAL_MEMORY",
+            "value": "1020054733"
+          },
+          {
+            "name": "ARANGODB_ZONE",
+            "value": "0"
+          },
+          {
+            "name": "ARANGODB_OVERRIDE_SERVER_GROUP",
+            "value": "coordinator"
+          },
+          {
+            "name": "ARANGODB_OVERRIDE_DEPLOYMENT_MODE",
+            "value": "Cluster"
+          },
+          {
+            "name": "ARANGODB_OVERRIDE_VERSION",
+            "value": "3.9.2"
+          },
+          {
+            "name": "ARANGODB_OVERRIDE_ENTERPRISE",
+            "value": "true"
+          }
+        ],
+        "envFrom": [
+          {
+            "configMapRef": {
+              "name": "arangodb-operator-feature-config-map",
+              "optional": true
+            }
+          }
+        ],
+        "image": "arangodb/arangodb",
+        "imagePullPolicy": "IfNotPresent",
+        "lifecycle": {
+          "preStop": {
+            "exec": {
+              "command": [
+                "/lifecycle/tools/arangodb_operator",
+                "lifecycle",
+                "preStop",
+                "port"
+              ]
+            }
+          }
+        },
+        "name": "server",
+        "ports": [
+          {
+            "containerPort": 8529,
+            "name": "server",
+            "protocol": "TCP"
+          }
+        ],
+        "readinessProbe": {
+          "exec": {
+            "command": [
+              "/lifecycle/tools/arangodb_operator",
+              "lifecycle",
+              "probe",
+              "readiness",
+              "--ssl",
+              "--auth"
+            ]
+          },
+          "failureThreshold": 10,
+          "initialDelaySeconds": 2,
+          "periodSeconds": 2,
+          "successThreshold": 1,
+          "timeoutSeconds": 2
+        },
+        "resources": {
+          "limits": {
+            "cpu": "250m",
+            "memory": "1073741824"
+          },
+          "requests": {
+            "cpu": "250m",
+            "memory": "1073741824"
+          }
+        },
+        "securityContext": {
+          "capabilities": {
+          }
+        },
+        "startupProbe": {
+          "exec": {
+            "command": [
+              "/lifecycle/tools/arangodb_operator",
+              "lifecycle",
+              "probe",
+              "startup",
+              "--ssl",
+              "--auth"
+            ]
+          },
+          "failureThreshold": 600,
+          "initialDelaySeconds": 1,
+          "periodSeconds": 5,
+          "successThreshold": 1,
+          "timeoutSeconds": 2
+        },
+        "volumeMounts": [
+          {
+            "mountPath": "/data",
+            "name": "arangod-data"
+          },
+          {
+            "mountPath": "/lifecycle/tools",
+            "name": "lifecycle"
+          },
+          {
+            "mountPath": "/secrets/tls",
+            "name": "tls-keyfile",
+            "readOnly": true
+          },
+          {
+            "mountPath": "/secrets/rocksdb/encryption",
+            "name": "rocksdb-encryption"
+          },
+          {
+            "mountPath": "/secrets/cluster/jwt",
+            "name": "cluster-jwt"
+          }
+        ]
+      }
+    ],
+    "hostname": "deployment-coordinator-0000000",
+    "initContainers": [
+      {
+        "command": [
+          "/usr/bin/arangodb_operator",
+          "lifecycle",
+          "copy",
+          "--target",
+          "/lifecycle/tools"
+        ],
+        "image": "arangodb/kube-arangodb",
+        "imagePullPolicy": "IfNotPresent",
+        "name": "init-lifecycle",
+        "resources": {
+        },
+        "securityContext": {
+          "capabilities": {
+          }
+        },
+        "volumeMounts": [
+          {
+            "mountPath": "/lifecycle/tools",
+            "name": "lifecycle"
+          }
+        ]
+      },
+      {
+        "command": [
+          "/usr/bin/arangodb_operator",
+          "uuid",
+          "--uuid-path",
+          "/data/UUID",
+          "--engine-path",
+          "/data/ENGINE",
+          "--uuid",
+          "CRDN-0000000",
+          "--engine",
+          "rocksdb"
+        ],
+        "env": [
+          {
+            "name": "MY_POD_NAMESPACE",
+            "value": "depl-eayebbnrei1ao4ewwaac"
+          }
+        ],
+        "image": "arangodb/kube-arangodb",
+        "name": "uuid",
+        "resources": {
+          "limits": {
+            "cpu": "100m",
+            "memory": "50Mi"
+          },
+          "requests": {
+            "cpu": "100m",
+            "memory": "10Mi"
+          }
+        },
+        "securityContext": {
+          "capabilities": {
+          }
+        },
+        "volumeMounts": [
+          {
+            "mountPath": "/data",
+            "name": "arangod-data"
+          }
+        ]
+      }
+    ],
+    "restartPolicy": "Never",
+    "serviceAccountName": "deployment-pod",
+    "subdomain": "deployment-int",
+    "terminationGracePeriodSeconds": 3600,
+    "tolerations": [
+      {
+        "effect": "NoSchedule",
+        "key": "node.arangodb.com/out-of-volumes",
+        "operator": "Exists"
+      },
+      {
+        "effect": "NoExecute",
+        "key": "node.kubernetes.io/not-ready",
+        "operator": "Exists",
+        "tolerationSeconds": 15
+      },
+      {
+        "effect": "NoExecute",
+        "key": "node.kubernetes.io/unreachable",
+        "operator": "Exists",
+        "tolerationSeconds": 15
+      },
+      {
+        "effect": "NoExecute",
+        "key": "node.alpha.kubernetes.io/unreachable",
+        "operator": "Exists",
+        "tolerationSeconds": 15
+      }
+    ],
+    "volumes": [
+      {
+        "emptyDir": {
+        },
+        "name": "arangod-data"
+      },
+      {
+        "name": "tls-keyfile",
+        "secret": {
+          "secretName": "deployment-coordinator-0000000-tls-keyfile"
+        }
+      },
+      {
+        "name": "rocksdb-encryption",
+        "secret": {
+          "secretName": "deployment-rocksdb-encryption"
+        }
+      },
+      {
+        "name": "cluster-jwt",
+        "secret": {
+          "secretName": "deployment-jwt-folder"
+        }
+      },
+      {
+        "emptyDir": {
+        },
+        "name": "lifecycle"
+      }
+    ]
+  }
+}

pkg/deployment/rotation/testdata/pod_lifecycle_change.000.status.json

@@ -0,0 +1,351 @@
+{
+  "metadata": {
+    "creationTimestamp": null,
+    "finalizers": [
+      "pod.database.arangodb.com/delay",
+      "database.arangodb.com/graceful-shutdown"
+    ],
+    "labels": {
+      "app": "arangodb",
+      "arango_deployment": "deployment",
+      "deployment.arangodb.com/member": "CRDN-0000000",
+      "role": "coordinator"
+    },
+    "name": "deployment-crdn-0000000-49237a"
+  },
+  "spec": {
+    "containers": [
+      {
+        "command": [
+          "/usr/sbin/arangod",
+          "--cluster.agency-endpoint=ssl://10.0.111.156:8529",
+          "--cluster.agency-endpoint=ssl://10.0.142.55:8529",
+          "--cluster.agency-endpoint=ssl://10.0.208.123:8529",
+          "--cluster.my-address=ssl://10.0.238.70:8529",
+          "--cluster.my-role=COORDINATOR",
+          "--database.directory=/data",
+          "--foxx.queues=true",
+          "--log.level=INFO",
+          "--log.output=+",
+          "--rocksdb.encryption-keyfile=/secrets/rocksdb/encryption/key",
+          "--server.authentication=true",
+          "--server.endpoint=ssl://[::]:8529",
+          "--server.jwt-secret-folder=/secrets/cluster/jwt",
+          "--server.statistics=true",
+          "--server.storage-engine=rocksdb",
+          "--ssl.ecdh-curve=",
+          "--ssl.keyfile=/secrets/tls/tls.keyfile",
+          "--log.thread=true",
+          "--log.thread-name=true",
+          "--server.authentication-system-only=false",
+          "--javascript.startup-options-denylist=.*",
+          "--javascript.endpoints-allowlist=example.com",
+          "--javascript.environment-variables-allowlist=^HOSTNAME$",
+          "--javascript.environment-variables-allowlist=^PATH$",
+          "--javascript.files-allowlist=^$",
+          "--javascript.harden",
+          "--server.harden",
+          "--javascript.files-allowlist=^(?!(\\/bin\/.*))(?!(\/dev\/.*))(?!(\/etc\/.*))(?!(\/lib\/.*))(?!(\/lifecycle\/.*))(?!(\/media\/.*))(?!(\/mnt\/.*))(?!(\/opt\/.*))(?!(\/proc\/.*))(?!(\/root\/.*))(?!(\/run\/.*))(?!(\/sbin\/.*))(?!(\/secrets\/.*))(?!(\/srv\/.*))(?!(\/usr\/.*))(?!(\/var\/.*)).+",
+          "--backup.api-enabled=jwt",
+          "--cluster.max-number-of-shards=1",
+          "--cluster.min-replication-factor=3",
+          "--cluster.max-replication-factor=3",
+          "--cluster.upgrade=online",
+          "--server.export-metrics-api=true",
+          "--cluster.force-one-shard=true"
+        ],
+        "env": [
+          {
+            "name": "MY_POD_NAME",
+            "valueFrom": {
+              "fieldRef": {
+                "fieldPath": "metadata.name"
+              }
+            }
+          },
+          {
+            "name": "MY_POD_NAMESPACE",
+            "valueFrom": {
+              "fieldRef": {
+                "fieldPath": "metadata.namespace"
+              }
+            }
+          },
+          {
+            "name": "MY_NODE_NAME",
+            "valueFrom": {
+              "fieldRef": {
+                "fieldPath": "spec.nodeName"
+              }
+            }
+          },
+          {
+            "name": "NODE_NAME",
+            "valueFrom": {
+              "fieldRef": {
+                "fieldPath": "spec.nodeName"
+              }
+            }
+          },
+          {
+            "name": "ARANGODB_OVERRIDE_DETECTED_NUMBER_OF_CORES",
+            "value": "1"
+          },
+          {
+            "name": "ARANGODB_OVERRIDE_DETECTED_TOTAL_MEMORY",
+            "value": "1020054733"
+          },
+          {
+            "name": "ARANGODB_ZONE",
+            "value": "0"
+          },
+          {
+            "name": "ARANGODB_OVERRIDE_SERVER_GROUP",
+            "value": "coordinator"
+          },
+          {
+            "name": "ARANGODB_OVERRIDE_DEPLOYMENT_MODE",
+            "value": "Cluster"
+          },
+          {
+            "name": "ARANGODB_OVERRIDE_VERSION",
+            "value": "3.9.2"
+          },
+          {
+            "name": "ARANGODB_OVERRIDE_ENTERPRISE",
+            "value": "true"
+          }
+        ],
+        "envFrom": [
+          {
+            "configMapRef": {
+              "name": "arangodb-operator-feature-config-map",
+              "optional": true
+            }
+          }
+        ],
+        "image": "arangodb/arangodb",
+        "imagePullPolicy": "IfNotPresent",
+        "lifecycle": {
+          "preStop": {
+            "exec": {
+              "command": [
+                "/lifecycle/tools/arangodb_operator",
+                "lifecycle",
+                "preStop",
+                "port"
+              ]
+            }
+          }
+        },
+        "name": "server",
+        "ports": [
+          {
+            "containerPort": 8529,
+            "name": "server",
+            "protocol": "TCP"
+          }
+        ],
+        "readinessProbe": {
+          "exec": {
+            "command": [
+              "/lifecycle/tools/arangodb_operator",
+              "lifecycle",
+              "probe",
+              "--endpoint=/_admin/server/availability",
+              "--ssl",
+              "--auth"
+            ]
+          },
+          "failureThreshold": 10,
+          "initialDelaySeconds": 2,
+          "periodSeconds": 2,
+          "successThreshold": 1,
+          "timeoutSeconds": 2
+        },
+        "resources": {
+          "limits": {
+            "cpu": "250m",
+            "memory": "1073741824"
+          },
+          "requests": {
+            "cpu": "250m",
+            "memory": "1073741824"
+          }
+        },
+        "securityContext": {
+          "capabilities": {
+          }
+        },
+        "startupProbe": {
+          "exec": {
+            "command": [
+              "/lifecycle/tools/arangodb_operator",
+              "lifecycle",
+              "probe",
+              "startup",
+              "--ssl",
+              "--auth"
+            ]
+          },
+          "failureThreshold": 600,
+          "initialDelaySeconds": 1,
+          "periodSeconds": 5,
+          "successThreshold": 1,
+          "timeoutSeconds": 2
+        },
+        "volumeMounts": [
+          {
+            "mountPath": "/data",
+            "name": "arangod-data"
+          },
+          {
+            "mountPath": "/lifecycle/tools",
+            "name": "lifecycle"
+          },
+          {
+            "mountPath": "/secrets/tls",
+            "name": "tls-keyfile",
+            "readOnly": true
+          },
+          {
+            "mountPath": "/secrets/rocksdb/encryption",
+            "name": "rocksdb-encryption"
+          },
+          {
+            "mountPath": "/secrets/cluster/jwt",
+            "name": "cluster-jwt"
+          }
+        ]
+      }
+    ],
+    "hostname": "deployment-coordinator-0000000",
+    "initContainers": [
+      {
+        "command": [
+          "/usr/bin/arangodb_operator",
+          "lifecycle",
+          "copy",
+          "--target",
+          "/lifecycle/tools"
+        ],
+        "image": "arangodb/kube-arangodb",
+        "imagePullPolicy": "IfNotPresent",
+        "name": "init-lifecycle",
+        "resources": {
+        },
+        "securityContext": {
+          "capabilities": {
+          }
+        },
+        "volumeMounts": [
+          {
+            "mountPath": "/lifecycle/tools",
+            "name": "lifecycle"
+          }
+        ]
+      },
+      {
+        "command": [
+          "/usr/bin/arangodb_operator",
+          "uuid",
+          "--uuid-path",
+          "/data/UUID",
+          "--engine-path",
+          "/data/ENGINE",
+          "--uuid",
+          "CRDN-0000000",
+          "--engine",
+          "rocksdb"
+        ],
+        "env": [
+          {
+            "name": "MY_POD_NAMESPACE",
+            "value": "depl-eayebbnrei1ao4ewwaac"
+          }
+        ],
+        "image": "arangodb/kube-arangodb",
+        "name": "uuid",
+        "resources": {
+          "limits": {
+            "cpu": "100m",
+            "memory": "50Mi"
+          },
+          "requests": {
+            "cpu": "100m",
+            "memory": "10Mi"
+          }
+        },
+        "securityContext": {
+          "capabilities": {
+          }
+        },
+        "volumeMounts": [
+          {
+            "mountPath": "/data",
+            "name": "arangod-data"
+          }
+        ]
+      }
+    ],
+    "restartPolicy": "Never",
+    "serviceAccountName": "deployment-pod",
+    "subdomain": "deployment-int",
+    "terminationGracePeriodSeconds": 3600,
+    "tolerations": [
+      {
+        "effect": "NoSchedule",
+        "key": "node.arangodb.com/out-of-volumes",
+        "operator": "Exists"
+      },
+      {
+        "effect": "NoExecute",
+        "key": "node.kubernetes.io/not-ready",
+        "operator": "Exists",
+        "tolerationSeconds": 15
+      },
+      {
+        "effect": "NoExecute",
+        "key": "node.kubernetes.io/unreachable",
+        "operator": "Exists",
+        "tolerationSeconds": 15
+      },
+      {
+        "effect": "NoExecute",
+        "key": "node.alpha.kubernetes.io/unreachable",
+        "operator": "Exists",
+        "tolerationSeconds": 15
+      }
+    ],
+    "volumes": [
+      {
+        "emptyDir": {
+        },
+        "name": "arangod-data"
+      },
+      {
+        "name": "tls-keyfile",
+        "secret": {
+          "secretName": "deployment-coordinator-0000000-tls-keyfile"
+        }
+      },
+      {
+        "name": "rocksdb-encryption",
+        "secret": {
+          "secretName": "deployment-rocksdb-encryption"
+        }
+      },
+      {
+        "name": "cluster-jwt",
+        "secret": {
+          "secretName": "deployment-jwt-folder"
+        }
+      },
+      {
+        "emptyDir": {
+        },
+        "name": "lifecycle"
+      }
+    ]
+  }
+}

pkg/util/k8sutil/lifecycle.go

@@ -36,6 +36,26 @@ const (
 	lifecycleVolumeName        = "lifecycle"
 )
 
+var (
+	binaryPath string
+)
+
+func init() {
+	if b, err := os.Executable(); err != nil {
+		panic(err.Error())
+	} else {
+		binaryPath = b
+	}
+}
+
+func SetBinaryPath(path string) {
+	binaryPath = path
+}
+
+func LifecycleBinary() string {
+	return filepath.Join(LifecycleVolumeMountDir, filepath.Base(binaryPath))
+}
+
 // InitLifecycleContainer creates an init-container to copy the lifecycle binary to a shared volume.
 func InitLifecycleContainer(image string, resources *core.ResourceRequirements, securityContext *core.SecurityContext) (core.Container, error) {
 	binaryPath, err := os.Executable()
@@ -71,11 +91,7 @@ func NewLifecyclePort() (*core.Lifecycle, error) {
 
 // NewLifecycle creates a lifecycle structure with preStop handler.
 func NewLifecycle(t string) (*core.Lifecycle, error) {
-	binaryPath, err := os.Executable()
+	exePath := LifecycleBinary()
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-	exePath := filepath.Join(LifecycleVolumeMountDir, filepath.Base(binaryPath))
 	lifecycle := &core.Lifecycle{
 		PreStop: &core.Handler{
 			Exec: &core.ExecAction{