From 198cac593f056217ed592458f2f1b46f853e5ed1 Mon Sep 17 00:00:00 2001 From: Adam Janikowski <12255597+ajanikow@users.noreply.github.com> Date: Tue, 27 Sep 2022 16:11:57 +0200 Subject: [PATCH] [Bugfix] Prevent LifeCycle restarts (#1127) --- CHANGELOG.md | 1 + go.mod | 11 +- go.sum | 12 + pkg/deployment/resources/internal_exporter.go | 10 +- .../resources/pod_creator_probes.go | 28 +- pkg/deployment/rotation/arangod_containers.go | 20 +- pkg/deployment/rotation/arangod_probes.go | 65 ++++ pkg/deployment/rotation/compare.go | 28 +- pkg/deployment/rotation/logger.go | 27 ++ pkg/deployment/rotation/predefined_test.go | 76 ++++ .../pod_lifecycle_change.000.spec.json | 351 ++++++++++++++++++ .../pod_lifecycle_change.000.status.json | 351 ++++++++++++++++++ pkg/util/k8sutil/lifecycle.go | 26 +- 13 files changed, 952 insertions(+), 54 deletions(-) create mode 100644 pkg/deployment/rotation/arangod_probes.go create mode 100644 pkg/deployment/rotation/logger.go create mode 100644 pkg/deployment/rotation/predefined_test.go create mode 100644 pkg/deployment/rotation/testdata/pod_lifecycle_change.000.spec.json create mode 100644 pkg/deployment/rotation/testdata/pod_lifecycle_change.000.status.json diff --git a/CHANGELOG.md b/CHANGELOG.md index dde45162f..c1122b8e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - (Feature) Add Member Update helpers - (Feature) Active Member condition - (Bugfix) Accept Initial Spec +- (Bugfix) Prevent LifeCycle restarts ## [1.2.17](https://github.com/arangodb/kube-arangodb/tree/1.2.17) (2022-09-22) - (Feature) Add new field to DeploymentReplicationStatus with details on DC2DC sync status= diff --git a/go.mod b/go.mod index a7d1285dc..cf9ae080a 100644 --- a/go.mod +++ b/go.mod @@ -55,9 +55,6 @@ require ( k8s.io/apimachinery v0.21.10 k8s.io/client-go v12.0.0+incompatible k8s.io/klog v1.0.0 -) - -require ( github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect @@ -106,6 +103,10 @@ require ( k8s.io/utils v0.0.0-20210521133846-da695404a2bc // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect sigs.k8s.io/yaml v1.2.0 + github.com/josephburnett/jd v1.6.1 + github.com/cenkalti/backoff/v4 v4.1.3 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/swag v0.21.1 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect ) - -require github.com/cenkalti/backoff/v4 v4.1.3 // indirect diff --git a/go.sum b/go.sum index c3d35847b..bf72eb153 100644 --- a/go.sum +++ b/go.sum @@ -164,12 +164,16 @@ github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= @@ -305,6 +309,10 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/jessevdk/go-assets v0.0.0-20160921144138-4f4301a06e15 h1:cW/amwGEJK5MSKntPXRjX4dxs/nGxGT8gXKIsKFmHGc= github.com/jessevdk/go-assets v0.0.0-20160921144138-4f4301a06e15/go.mod h1:Fdm/oWRW+CH8PRbLntksCNtmcCBximKPkVQYvmMl80k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/josephburnett/jd v1.6.1 h1:Uzqhcje4WqvVyp85F3Oj0ezISPTlnhnr/KaLZIy8qh0= +github.com/josephburnett/jd v1.6.1/go.mod h1:R8ZnZnLt2D4rhW4NvBc/USTo6mzyNT6fYNIIWOJA9GY= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -339,6 +347,9 @@ github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPK github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -900,6 +911,7 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= diff --git a/pkg/deployment/resources/internal_exporter.go b/pkg/deployment/resources/internal_exporter.go index 1ca6b498a..9f1c5a19c 100644 --- a/pkg/deployment/resources/internal_exporter.go +++ b/pkg/deployment/resources/internal_exporter.go @@ -19,14 +19,10 @@ package resources import ( - "os" - "path/filepath" - core "k8s.io/api/core/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/apis/shared" - "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes" ) @@ -36,11 +32,7 @@ func ArangodbInternalExporterContainer(image string, args []string, livenessProb resources core.ResourceRequirements, securityContext *core.SecurityContext, spec api.DeploymentSpec) (core.Container, error) { - binaryPath, err := os.Executable() - if err != nil { - return core.Container{}, errors.WithStack(err) - } - exePath := filepath.Join(k8sutil.LifecycleVolumeMountDir, filepath.Base(binaryPath)) + exePath := k8sutil.LifecycleBinary() c := core.Container{ Name: shared.ExporterContainerName, diff --git a/pkg/deployment/resources/pod_creator_probes.go b/pkg/deployment/resources/pod_creator_probes.go index 1de7ea951..5c7eef0fa 100644 --- a/pkg/deployment/resources/pod_creator_probes.go +++ b/pkg/deployment/resources/pod_creator_probes.go @@ -22,8 +22,6 @@ package resources import ( "math" - "os" - "path/filepath" "time" core "k8s.io/api/core/v1" @@ -223,18 +221,13 @@ func (r *Resources) probeBuilders(imageInfo api.ImageInfo) map[api.ServerGroup]p } } -func (r *Resources) probeCommand(spec api.DeploymentSpec, probeType api.ProbeType) ([]string, error) { - binaryPath, err := os.Executable() - if err != nil { - return nil, err - } - exePath := filepath.Join(k8sutil.LifecycleVolumeMountDir, filepath.Base(binaryPath)) +func (r *Resources) probeCommand(spec api.DeploymentSpec, probeType api.ProbeType) []string { + exePath := k8sutil.LifecycleBinary() args := []string{ exePath, "lifecycle", "probe", string(probeType), - // TODO test rotation required when changed } if spec.IsSecure() { @@ -245,7 +238,7 @@ func (r *Resources) probeCommand(spec api.DeploymentSpec, probeType api.ProbeTyp args = append(args, "--auth") } - return args, nil + return args } func (r *Resources) probeBuilderLivenessCoreSelect(group api.ServerGroup, imageInfo api.ImageInfo) probeBuilder { @@ -266,10 +259,7 @@ func (r *Resources) probeBuilderStartupCoreSelect(group api.ServerGroup, imageIn func (r *Resources) probeBuilderLivenessCoreOperator(spec api.DeploymentSpec, group api.ServerGroup, image api.ImageInfo) (Probe, error) { - args, err := r.probeCommand(spec, api.ProbeTypeLiveness) - if err != nil { - return nil, err - } + args := r.probeCommand(spec, api.ProbeTypeLiveness) cmdProbeConfig := &probes.CMDProbeConfig{ Command: args, @@ -283,10 +273,7 @@ func (r *Resources) probeBuilderLivenessCoreOperator(spec api.DeploymentSpec, gr func (r *Resources) probeBuilderStartupCoreOperator(spec api.DeploymentSpec, group api.ServerGroup, image api.ImageInfo) (Probe, error) { - args, err := r.probeCommand(spec, api.ProbeTypeStartUp) - if err != nil { - return nil, err - } + args := r.probeCommand(spec, api.ProbeTypeStartUp) retries, periodSeconds := getProbeRetries(group) if IsServerProgressAvailable(group, image) { @@ -399,10 +386,7 @@ func (r *Resources) probeBuilderReadinessCoreSelect() probeBuilder { } func (r *Resources) probeBuilderReadinessCoreOperator(spec api.DeploymentSpec, _ api.ServerGroup, _ api.ImageInfo) (Probe, error) { - args, err := r.probeCommand(spec, api.ProbeTypeReadiness) - if err != nil { - return nil, err - } + args := r.probeCommand(spec, api.ProbeTypeReadiness) return &probes.CMDProbeConfig{ Command: args, diff --git a/pkg/deployment/rotation/arangod_containers.go b/pkg/deployment/rotation/arangod_containers.go index 1e43c882e..943108c69 100644 --- a/pkg/deployment/rotation/arangod_containers.go +++ b/pkg/deployment/rotation/arangod_containers.go @@ -32,6 +32,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/deployment/resources" "github.com/arangodb/kube-arangodb/pkg/deployment/topology" "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) const ( @@ -61,7 +62,7 @@ func containersCompare(ds api.DeploymentSpec, g api.ServerGroup, spec, status *c g := podContainerFuncGenerator(ds, g, ac, bc) - if m, p, err := comparePodContainer(builder, g(compareServerContainerVolumeMounts)); err != nil { + if m, p, err := comparePodContainer(builder, g(compareServerContainerVolumeMounts), g(compareServerContainerProbes)); err != nil { log.Err(err).Msg("Error while getting pod diff") return SkippedRotation, nil, err } else { @@ -104,11 +105,6 @@ func containersCompare(ds api.DeploymentSpec, g api.ServerGroup, spec, status *c bc.Ports = ac.Ports mode = mode.And(SilentRotation) } - - if !areProbesEqual(ac.StartupProbe, bc.StartupProbe) { - bc.StartupProbe = ac.StartupProbe - mode = mode.And(SilentRotation) - } } else { if ac.Image != bc.Image { // Image changed @@ -252,6 +248,18 @@ func areProbesEqual(a, b *core.Probe) bool { return equality.Semantic.DeepEqual(a, b) } +func isManagedProbe(a, b *core.Probe) bool { + if a.Exec == nil || b.Exec == nil { + return false + } + + if len(a.Exec.Command) == 0 || len(b.Exec.Command) == 0 { + return false + } + + return a.Exec.Command[0] == k8sutil.LifecycleBinary() && b.Exec.Command[0] == k8sutil.LifecycleBinary() +} + // areEnvsFromEqual returns true when environment variables from source are the same after filtering. func areEnvsFromEqual(a, b []core.EnvFromSource, rules ...func(a, b map[string]core.EnvFromSource) (map[string]core.EnvFromSource, map[string]core.EnvFromSource)) bool { am := createEnvsFromMap(a) diff --git a/pkg/deployment/rotation/arangod_probes.go b/pkg/deployment/rotation/arangod_probes.go new file mode 100644 index 000000000..c3c559cb3 --- /dev/null +++ b/pkg/deployment/rotation/arangod_probes.go @@ -0,0 +1,65 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package rotation + +import ( + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" +) + +func compareServerContainerProbes(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.Container) comparePodContainerFunc { + return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) { + if !areProbesEqual(spec.StartupProbe, status.StartupProbe) { + status.StartupProbe = spec.StartupProbe + mode = mode.And(SilentRotation) + } + + if !areProbesEqual(spec.ReadinessProbe, status.ReadinessProbe) { + if isManagedProbe(spec.ReadinessProbe, status.ReadinessProbe) { + q := status.ReadinessProbe.DeepCopy() + + q.Exec = spec.ReadinessProbe.Exec.DeepCopy() + + if equality.Semantic.DeepDerivative(spec.ReadinessProbe, q) { + status.ReadinessProbe = spec.ReadinessProbe + mode = mode.And(SilentRotation) + } + } + } + + if !areProbesEqual(spec.LivenessProbe, status.LivenessProbe) { + if isManagedProbe(spec.LivenessProbe, status.LivenessProbe) { + q := status.LivenessProbe.DeepCopy() + + q.Exec = spec.LivenessProbe.Exec.DeepCopy() + + if equality.Semantic.DeepDerivative(spec.LivenessProbe, q) { + status.LivenessProbe = spec.LivenessProbe + mode = mode.And(SilentRotation) + } + } + } + + return + } +} diff --git a/pkg/deployment/rotation/compare.go b/pkg/deployment/rotation/compare.go index a5d0ffcb5..b771bd320 100644 --- a/pkg/deployment/rotation/compare.go +++ b/pkg/deployment/rotation/compare.go @@ -23,6 +23,7 @@ package rotation import ( "encoding/json" + jd "github.com/josephburnett/jd/lib" "github.com/rs/zerolog/log" core "k8s.io/api/core/v1" @@ -114,14 +115,27 @@ func compare(deploymentSpec api.DeploymentSpec, member api.MemberStatus, group a } if spec.RotationNeeded(newStatus) { - specData, _ := json.Marshal(spec) - statusData, _ := json.Marshal(newStatus) + line := logger.Str("id", member.ID) - log.Info().Str("before", spec.PodSpecChecksum). - Str("id", member.ID). - Str("spec", string(specData)). - Str("status", string(statusData)). - Msg("Pod needs rotation - templates does not match") + specBytes, errA := json.Marshal(spec.PodSpec) + if errA == nil { + line = line.Str("spec", string(specBytes)) + } + + statusBytes, errB := json.Marshal(newStatus.PodSpec) + if errB == nil { + line = line.Str("status", string(statusBytes)) + } + + if errA == nil && errB == nil { + if specData, err := jd.ReadJsonString(string(specBytes)); err == nil && specData != nil { + if statusData, err := jd.ReadJsonString(string(statusBytes)); err == nil && statusData != nil { + line = line.Str("diff", specData.Diff(statusData).Render()) + } + } + } + + line.Info("Pod needs rotation - templates does not match") return GracefulRotation, nil, nil } diff --git a/pkg/deployment/rotation/logger.go b/pkg/deployment/rotation/logger.go new file mode 100644 index 000000000..352d11122 --- /dev/null +++ b/pkg/deployment/rotation/logger.go @@ -0,0 +1,27 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package rotation + +import "github.com/arangodb/kube-arangodb/pkg/logging" + +var ( + logger = logging.Global().RegisterAndGetLogger("pod_compare", logging.Info) +) diff --git a/pkg/deployment/rotation/predefined_test.go b/pkg/deployment/rotation/predefined_test.go new file mode 100644 index 000000000..fea76c91b --- /dev/null +++ b/pkg/deployment/rotation/predefined_test.go @@ -0,0 +1,76 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package rotation + +import ( + _ "embed" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + core "k8s.io/api/core/v1" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/resources" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +func init() { + k8sutil.SetBinaryPath("arangodb_operator") +} + +//go:embed testdata/pod_lifecycle_change.000.spec.json +var podLifecycleChange000Spec []byte + +//go:embed testdata/pod_lifecycle_change.000.status.json +var podLifecycleChange000Status []byte + +func runPredefinedTests(t *testing.T, spec, status []byte) (mode Mode, plan api.Plan, err error) { + var specO, statusO core.PodTemplateSpec + + require.NoError(t, json.Unmarshal(spec, &specO)) + require.NoError(t, json.Unmarshal(status, &statusO)) + + specC, err := resources.ChecksumArangoPod(api.ServerGroupSpec{}, resources.CreatePodFromTemplate(&specO)) + require.NoError(t, err) + + statusC, err := resources.ChecksumArangoPod(api.ServerGroupSpec{}, resources.CreatePodFromTemplate(&statusO)) + require.NoError(t, err) + + obj := api.DeploymentSpec{} + member := api.MemberStatus{} + + specT, err := api.GetArangoMemberPodTemplate(&specO, specC) + require.NoError(t, err) + statusT, err := api.GetArangoMemberPodTemplate(&statusO, statusC) + require.NoError(t, err) + + return compare(obj, member, api.ServerGroupUnknown, specT, statusT) +} + +func Test_PredefinedTests(t *testing.T) { + t.Run("podLifecycleChange000", func(t *testing.T) { + mode, plan, err := runPredefinedTests(t, podLifecycleChange000Spec, podLifecycleChange000Status) + require.NoError(t, err) + require.Empty(t, plan) + require.Equal(t, SilentRotation, mode) + }) +} diff --git a/pkg/deployment/rotation/testdata/pod_lifecycle_change.000.spec.json b/pkg/deployment/rotation/testdata/pod_lifecycle_change.000.spec.json new file mode 100644 index 000000000..b33d499e0 --- /dev/null +++ b/pkg/deployment/rotation/testdata/pod_lifecycle_change.000.spec.json @@ -0,0 +1,351 @@ +{ + "metadata": { + "creationTimestamp": null, + "finalizers": [ + "pod.database.arangodb.com/delay", + "database.arangodb.com/graceful-shutdown" + ], + "labels": { + "app": "arangodb", + "arango_deployment": "deployment", + "deployment.arangodb.com/member": "CRDN-0000000", + "role": "coordinator" + }, + "name": "deployment-crdn-0000000-49237a" + }, + "spec": { + "containers": [ + { + "command": [ + "/usr/sbin/arangod", + "--cluster.agency-endpoint=ssl://10.0.111.156:8529", + "--cluster.agency-endpoint=ssl://10.0.142.55:8529", + "--cluster.agency-endpoint=ssl://10.0.208.123:8529", + "--cluster.my-address=ssl://10.0.238.70:8529", + "--cluster.my-role=COORDINATOR", + "--database.directory=/data", + "--foxx.queues=true", + "--log.level=INFO", + "--log.output=+", + "--rocksdb.encryption-keyfile=/secrets/rocksdb/encryption/key", + "--server.authentication=true", + "--server.endpoint=ssl://[::]:8529", + "--server.jwt-secret-folder=/secrets/cluster/jwt", + "--server.statistics=true", + "--server.storage-engine=rocksdb", + "--ssl.ecdh-curve=", + "--ssl.keyfile=/secrets/tls/tls.keyfile", + "--log.thread=true", + "--log.thread-name=true", + "--server.authentication-system-only=false", + "--javascript.startup-options-denylist=.*", + "--javascript.endpoints-allowlist=example.com", + "--javascript.environment-variables-allowlist=^HOSTNAME$", + "--javascript.environment-variables-allowlist=^PATH$", + "--javascript.files-allowlist=^$", + "--javascript.harden", + "--server.harden", + "--javascript.files-allowlist=^(?!(\\/bin\/.*))(?!(\/dev\/.*))(?!(\/etc\/.*))(?!(\/lib\/.*))(?!(\/lifecycle\/.*))(?!(\/media\/.*))(?!(\/mnt\/.*))(?!(\/opt\/.*))(?!(\/proc\/.*))(?!(\/root\/.*))(?!(\/run\/.*))(?!(\/sbin\/.*))(?!(\/secrets\/.*))(?!(\/srv\/.*))(?!(\/usr\/.*))(?!(\/var\/.*)).+", + "--backup.api-enabled=jwt", + "--cluster.max-number-of-shards=1", + "--cluster.min-replication-factor=3", + "--cluster.max-replication-factor=3", + "--cluster.upgrade=online", + "--server.export-metrics-api=true", + "--cluster.force-one-shard=true" + ], + "env": [ + { + "name": "MY_POD_NAME", + "valueFrom": { + "fieldRef": { + "fieldPath": "metadata.name" + } + } + }, + { + "name": "MY_POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "MY_NODE_NAME", + "valueFrom": { + "fieldRef": { + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "ARANGODB_OVERRIDE_DETECTED_NUMBER_OF_CORES", + "value": "1" + }, + { + "name": "ARANGODB_OVERRIDE_DETECTED_TOTAL_MEMORY", + "value": "1020054733" + }, + { + "name": "ARANGODB_ZONE", + "value": "0" + }, + { + "name": "ARANGODB_OVERRIDE_SERVER_GROUP", + "value": "coordinator" + }, + { + "name": "ARANGODB_OVERRIDE_DEPLOYMENT_MODE", + "value": "Cluster" + }, + { + "name": "ARANGODB_OVERRIDE_VERSION", + "value": "3.9.2" + }, + { + "name": "ARANGODB_OVERRIDE_ENTERPRISE", + "value": "true" + } + ], + "envFrom": [ + { + "configMapRef": { + "name": "arangodb-operator-feature-config-map", + "optional": true + } + } + ], + "image": "arangodb/arangodb", + "imagePullPolicy": "IfNotPresent", + "lifecycle": { + "preStop": { + "exec": { + "command": [ + "/lifecycle/tools/arangodb_operator", + "lifecycle", + "preStop", + "port" + ] + } + } + }, + "name": "server", + "ports": [ + { + "containerPort": 8529, + "name": "server", + "protocol": "TCP" + } + ], + "readinessProbe": { + "exec": { + "command": [ + "/lifecycle/tools/arangodb_operator", + "lifecycle", + "probe", + "readiness", + "--ssl", + "--auth" + ] + }, + "failureThreshold": 10, + "initialDelaySeconds": 2, + "periodSeconds": 2, + "successThreshold": 1, + "timeoutSeconds": 2 + }, + "resources": { + "limits": { + "cpu": "250m", + "memory": "1073741824" + }, + "requests": { + "cpu": "250m", + "memory": "1073741824" + } + }, + "securityContext": { + "capabilities": { + } + }, + "startupProbe": { + "exec": { + "command": [ + "/lifecycle/tools/arangodb_operator", + "lifecycle", + "probe", + "startup", + "--ssl", + "--auth" + ] + }, + "failureThreshold": 600, + "initialDelaySeconds": 1, + "periodSeconds": 5, + "successThreshold": 1, + "timeoutSeconds": 2 + }, + "volumeMounts": [ + { + "mountPath": "/data", + "name": "arangod-data" + }, + { + "mountPath": "/lifecycle/tools", + "name": "lifecycle" + }, + { + "mountPath": "/secrets/tls", + "name": "tls-keyfile", + "readOnly": true + }, + { + "mountPath": "/secrets/rocksdb/encryption", + "name": "rocksdb-encryption" + }, + { + "mountPath": "/secrets/cluster/jwt", + "name": "cluster-jwt" + } + ] + } + ], + "hostname": "deployment-coordinator-0000000", + "initContainers": [ + { + "command": [ + "/usr/bin/arangodb_operator", + "lifecycle", + "copy", + "--target", + "/lifecycle/tools" + ], + "image": "arangodb/kube-arangodb", + "imagePullPolicy": "IfNotPresent", + "name": "init-lifecycle", + "resources": { + }, + "securityContext": { + "capabilities": { + } + }, + "volumeMounts": [ + { + "mountPath": "/lifecycle/tools", + "name": "lifecycle" + } + ] + }, + { + "command": [ + "/usr/bin/arangodb_operator", + "uuid", + "--uuid-path", + "/data/UUID", + "--engine-path", + "/data/ENGINE", + "--uuid", + "CRDN-0000000", + "--engine", + "rocksdb" + ], + "env": [ + { + "name": "MY_POD_NAMESPACE", + "value": "depl-eayebbnrei1ao4ewwaac" + } + ], + "image": "arangodb/kube-arangodb", + "name": "uuid", + "resources": { + "limits": { + "cpu": "100m", + "memory": "50Mi" + }, + "requests": { + "cpu": "100m", + "memory": "10Mi" + } + }, + "securityContext": { + "capabilities": { + } + }, + "volumeMounts": [ + { + "mountPath": "/data", + "name": "arangod-data" + } + ] + } + ], + "restartPolicy": "Never", + "serviceAccountName": "deployment-pod", + "subdomain": "deployment-int", + "terminationGracePeriodSeconds": 3600, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "node.arangodb.com/out-of-volumes", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 15 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 15 + }, + { + "effect": "NoExecute", + "key": "node.alpha.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 15 + } + ], + "volumes": [ + { + "emptyDir": { + }, + "name": "arangod-data" + }, + { + "name": "tls-keyfile", + "secret": { + "secretName": "deployment-coordinator-0000000-tls-keyfile" + } + }, + { + "name": "rocksdb-encryption", + "secret": { + "secretName": "deployment-rocksdb-encryption" + } + }, + { + "name": "cluster-jwt", + "secret": { + "secretName": "deployment-jwt-folder" + } + }, + { + "emptyDir": { + }, + "name": "lifecycle" + } + ] + } +} \ No newline at end of file diff --git a/pkg/deployment/rotation/testdata/pod_lifecycle_change.000.status.json b/pkg/deployment/rotation/testdata/pod_lifecycle_change.000.status.json new file mode 100644 index 000000000..117c9c3e5 --- /dev/null +++ b/pkg/deployment/rotation/testdata/pod_lifecycle_change.000.status.json @@ -0,0 +1,351 @@ +{ + "metadata": { + "creationTimestamp": null, + "finalizers": [ + "pod.database.arangodb.com/delay", + "database.arangodb.com/graceful-shutdown" + ], + "labels": { + "app": "arangodb", + "arango_deployment": "deployment", + "deployment.arangodb.com/member": "CRDN-0000000", + "role": "coordinator" + }, + "name": "deployment-crdn-0000000-49237a" + }, + "spec": { + "containers": [ + { + "command": [ + "/usr/sbin/arangod", + "--cluster.agency-endpoint=ssl://10.0.111.156:8529", + "--cluster.agency-endpoint=ssl://10.0.142.55:8529", + "--cluster.agency-endpoint=ssl://10.0.208.123:8529", + "--cluster.my-address=ssl://10.0.238.70:8529", + "--cluster.my-role=COORDINATOR", + "--database.directory=/data", + "--foxx.queues=true", + "--log.level=INFO", + "--log.output=+", + "--rocksdb.encryption-keyfile=/secrets/rocksdb/encryption/key", + "--server.authentication=true", + "--server.endpoint=ssl://[::]:8529", + "--server.jwt-secret-folder=/secrets/cluster/jwt", + "--server.statistics=true", + "--server.storage-engine=rocksdb", + "--ssl.ecdh-curve=", + "--ssl.keyfile=/secrets/tls/tls.keyfile", + "--log.thread=true", + "--log.thread-name=true", + "--server.authentication-system-only=false", + "--javascript.startup-options-denylist=.*", + "--javascript.endpoints-allowlist=example.com", + "--javascript.environment-variables-allowlist=^HOSTNAME$", + "--javascript.environment-variables-allowlist=^PATH$", + "--javascript.files-allowlist=^$", + "--javascript.harden", + "--server.harden", + "--javascript.files-allowlist=^(?!(\\/bin\/.*))(?!(\/dev\/.*))(?!(\/etc\/.*))(?!(\/lib\/.*))(?!(\/lifecycle\/.*))(?!(\/media\/.*))(?!(\/mnt\/.*))(?!(\/opt\/.*))(?!(\/proc\/.*))(?!(\/root\/.*))(?!(\/run\/.*))(?!(\/sbin\/.*))(?!(\/secrets\/.*))(?!(\/srv\/.*))(?!(\/usr\/.*))(?!(\/var\/.*)).+", + "--backup.api-enabled=jwt", + "--cluster.max-number-of-shards=1", + "--cluster.min-replication-factor=3", + "--cluster.max-replication-factor=3", + "--cluster.upgrade=online", + "--server.export-metrics-api=true", + "--cluster.force-one-shard=true" + ], + "env": [ + { + "name": "MY_POD_NAME", + "valueFrom": { + "fieldRef": { + "fieldPath": "metadata.name" + } + } + }, + { + "name": "MY_POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "MY_NODE_NAME", + "valueFrom": { + "fieldRef": { + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "ARANGODB_OVERRIDE_DETECTED_NUMBER_OF_CORES", + "value": "1" + }, + { + "name": "ARANGODB_OVERRIDE_DETECTED_TOTAL_MEMORY", + "value": "1020054733" + }, + { + "name": "ARANGODB_ZONE", + "value": "0" + }, + { + "name": "ARANGODB_OVERRIDE_SERVER_GROUP", + "value": "coordinator" + }, + { + "name": "ARANGODB_OVERRIDE_DEPLOYMENT_MODE", + "value": "Cluster" + }, + { + "name": "ARANGODB_OVERRIDE_VERSION", + "value": "3.9.2" + }, + { + "name": "ARANGODB_OVERRIDE_ENTERPRISE", + "value": "true" + } + ], + "envFrom": [ + { + "configMapRef": { + "name": "arangodb-operator-feature-config-map", + "optional": true + } + } + ], + "image": "arangodb/arangodb", + "imagePullPolicy": "IfNotPresent", + "lifecycle": { + "preStop": { + "exec": { + "command": [ + "/lifecycle/tools/arangodb_operator", + "lifecycle", + "preStop", + "port" + ] + } + } + }, + "name": "server", + "ports": [ + { + "containerPort": 8529, + "name": "server", + "protocol": "TCP" + } + ], + "readinessProbe": { + "exec": { + "command": [ + "/lifecycle/tools/arangodb_operator", + "lifecycle", + "probe", + "--endpoint=/_admin/server/availability", + "--ssl", + "--auth" + ] + }, + "failureThreshold": 10, + "initialDelaySeconds": 2, + "periodSeconds": 2, + "successThreshold": 1, + "timeoutSeconds": 2 + }, + "resources": { + "limits": { + "cpu": "250m", + "memory": "1073741824" + }, + "requests": { + "cpu": "250m", + "memory": "1073741824" + } + }, + "securityContext": { + "capabilities": { + } + }, + "startupProbe": { + "exec": { + "command": [ + "/lifecycle/tools/arangodb_operator", + "lifecycle", + "probe", + "startup", + "--ssl", + "--auth" + ] + }, + "failureThreshold": 600, + "initialDelaySeconds": 1, + "periodSeconds": 5, + "successThreshold": 1, + "timeoutSeconds": 2 + }, + "volumeMounts": [ + { + "mountPath": "/data", + "name": "arangod-data" + }, + { + "mountPath": "/lifecycle/tools", + "name": "lifecycle" + }, + { + "mountPath": "/secrets/tls", + "name": "tls-keyfile", + "readOnly": true + }, + { + "mountPath": "/secrets/rocksdb/encryption", + "name": "rocksdb-encryption" + }, + { + "mountPath": "/secrets/cluster/jwt", + "name": "cluster-jwt" + } + ] + } + ], + "hostname": "deployment-coordinator-0000000", + "initContainers": [ + { + "command": [ + "/usr/bin/arangodb_operator", + "lifecycle", + "copy", + "--target", + "/lifecycle/tools" + ], + "image": "arangodb/kube-arangodb", + "imagePullPolicy": "IfNotPresent", + "name": "init-lifecycle", + "resources": { + }, + "securityContext": { + "capabilities": { + } + }, + "volumeMounts": [ + { + "mountPath": "/lifecycle/tools", + "name": "lifecycle" + } + ] + }, + { + "command": [ + "/usr/bin/arangodb_operator", + "uuid", + "--uuid-path", + "/data/UUID", + "--engine-path", + "/data/ENGINE", + "--uuid", + "CRDN-0000000", + "--engine", + "rocksdb" + ], + "env": [ + { + "name": "MY_POD_NAMESPACE", + "value": "depl-eayebbnrei1ao4ewwaac" + } + ], + "image": "arangodb/kube-arangodb", + "name": "uuid", + "resources": { + "limits": { + "cpu": "100m", + "memory": "50Mi" + }, + "requests": { + "cpu": "100m", + "memory": "10Mi" + } + }, + "securityContext": { + "capabilities": { + } + }, + "volumeMounts": [ + { + "mountPath": "/data", + "name": "arangod-data" + } + ] + } + ], + "restartPolicy": "Never", + "serviceAccountName": "deployment-pod", + "subdomain": "deployment-int", + "terminationGracePeriodSeconds": 3600, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "node.arangodb.com/out-of-volumes", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 15 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 15 + }, + { + "effect": "NoExecute", + "key": "node.alpha.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 15 + } + ], + "volumes": [ + { + "emptyDir": { + }, + "name": "arangod-data" + }, + { + "name": "tls-keyfile", + "secret": { + "secretName": "deployment-coordinator-0000000-tls-keyfile" + } + }, + { + "name": "rocksdb-encryption", + "secret": { + "secretName": "deployment-rocksdb-encryption" + } + }, + { + "name": "cluster-jwt", + "secret": { + "secretName": "deployment-jwt-folder" + } + }, + { + "emptyDir": { + }, + "name": "lifecycle" + } + ] + } +} \ No newline at end of file diff --git a/pkg/util/k8sutil/lifecycle.go b/pkg/util/k8sutil/lifecycle.go index 0f3eda464..c075a10e5 100644 --- a/pkg/util/k8sutil/lifecycle.go +++ b/pkg/util/k8sutil/lifecycle.go @@ -36,6 +36,26 @@ const ( lifecycleVolumeName = "lifecycle" ) +var ( + binaryPath string +) + +func init() { + if b, err := os.Executable(); err != nil { + panic(err.Error()) + } else { + binaryPath = b + } +} + +func SetBinaryPath(path string) { + binaryPath = path +} + +func LifecycleBinary() string { + return filepath.Join(LifecycleVolumeMountDir, filepath.Base(binaryPath)) +} + // InitLifecycleContainer creates an init-container to copy the lifecycle binary to a shared volume. func InitLifecycleContainer(image string, resources *core.ResourceRequirements, securityContext *core.SecurityContext) (core.Container, error) { binaryPath, err := os.Executable() @@ -71,11 +91,7 @@ func NewLifecyclePort() (*core.Lifecycle, error) { // NewLifecycle creates a lifecycle structure with preStop handler. func NewLifecycle(t string) (*core.Lifecycle, error) { - binaryPath, err := os.Executable() - if err != nil { - return nil, errors.WithStack(err) - } - exePath := filepath.Join(LifecycleVolumeMountDir, filepath.Base(binaryPath)) + exePath := LifecycleBinary() lifecycle := &core.Lifecycle{ PreStop: &core.Handler{ Exec: &core.ExecAction{