1
0
Fork 0
mirror of https://github.com/arangodb/kube-arangodb.git synced 2024-12-14 11:57:37 +00:00

[Bugfix] Prevent LifeCycle restarts (#1127)

This commit is contained in:
Adam Janikowski 2022-09-27 16:11:57 +02:00 committed by GitHub
parent 87ea1ee032
commit 198cac593f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 952 additions and 54 deletions

View file

@ -5,6 +5,7 @@
- (Feature) Add Member Update helpers
- (Feature) Active Member condition
- (Bugfix) Accept Initial Spec
- (Bugfix) Prevent LifeCycle restarts
## [1.2.17](https://github.com/arangodb/kube-arangodb/tree/1.2.17) (2022-09-22)
- (Feature) Add new field to DeploymentReplicationStatus with details on DC2DC sync status=

11
go.mod
View file

@ -55,9 +55,6 @@ require (
k8s.io/apimachinery v0.21.10
k8s.io/client-go v12.0.0+incompatible
k8s.io/klog v1.0.0
)
require (
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
@ -106,6 +103,10 @@ require (
k8s.io/utils v0.0.0-20210521133846-da695404a2bc // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
sigs.k8s.io/yaml v1.2.0
github.com/josephburnett/jd v1.6.1
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/swag v0.21.1 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
)
require github.com/cenkalti/backoff/v4 v4.1.3 // indirect

12
go.sum
View file

@ -164,12 +164,16 @@ github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU=
github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
@ -305,6 +309,10 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
github.com/jessevdk/go-assets v0.0.0-20160921144138-4f4301a06e15 h1:cW/amwGEJK5MSKntPXRjX4dxs/nGxGT8gXKIsKFmHGc=
github.com/jessevdk/go-assets v0.0.0-20160921144138-4f4301a06e15/go.mod h1:Fdm/oWRW+CH8PRbLntksCNtmcCBximKPkVQYvmMl80k=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/josephburnett/jd v1.6.1 h1:Uzqhcje4WqvVyp85F3Oj0ezISPTlnhnr/KaLZIy8qh0=
github.com/josephburnett/jd v1.6.1/go.mod h1:R8ZnZnLt2D4rhW4NvBc/USTo6mzyNT6fYNIIWOJA9GY=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@ -339,6 +347,9 @@ github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPK
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@ -900,6 +911,7 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=

View file

@ -19,14 +19,10 @@
package resources
import (
"os"
"path/filepath"
core "k8s.io/api/core/v1"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/apis/shared"
"github.com/arangodb/kube-arangodb/pkg/util/errors"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes"
)
@ -36,11 +32,7 @@ func ArangodbInternalExporterContainer(image string, args []string, livenessProb
resources core.ResourceRequirements, securityContext *core.SecurityContext,
spec api.DeploymentSpec) (core.Container, error) {
binaryPath, err := os.Executable()
if err != nil {
return core.Container{}, errors.WithStack(err)
}
exePath := filepath.Join(k8sutil.LifecycleVolumeMountDir, filepath.Base(binaryPath))
exePath := k8sutil.LifecycleBinary()
c := core.Container{
Name: shared.ExporterContainerName,

View file

@ -22,8 +22,6 @@ package resources
import (
"math"
"os"
"path/filepath"
"time"
core "k8s.io/api/core/v1"
@ -223,18 +221,13 @@ func (r *Resources) probeBuilders(imageInfo api.ImageInfo) map[api.ServerGroup]p
}
}
func (r *Resources) probeCommand(spec api.DeploymentSpec, probeType api.ProbeType) ([]string, error) {
binaryPath, err := os.Executable()
if err != nil {
return nil, err
}
exePath := filepath.Join(k8sutil.LifecycleVolumeMountDir, filepath.Base(binaryPath))
func (r *Resources) probeCommand(spec api.DeploymentSpec, probeType api.ProbeType) []string {
exePath := k8sutil.LifecycleBinary()
args := []string{
exePath,
"lifecycle",
"probe",
string(probeType),
// TODO test rotation required when changed
}
if spec.IsSecure() {
@ -245,7 +238,7 @@ func (r *Resources) probeCommand(spec api.DeploymentSpec, probeType api.ProbeTyp
args = append(args, "--auth")
}
return args, nil
return args
}
func (r *Resources) probeBuilderLivenessCoreSelect(group api.ServerGroup, imageInfo api.ImageInfo) probeBuilder {
@ -266,10 +259,7 @@ func (r *Resources) probeBuilderStartupCoreSelect(group api.ServerGroup, imageIn
func (r *Resources) probeBuilderLivenessCoreOperator(spec api.DeploymentSpec, group api.ServerGroup,
image api.ImageInfo) (Probe, error) {
args, err := r.probeCommand(spec, api.ProbeTypeLiveness)
if err != nil {
return nil, err
}
args := r.probeCommand(spec, api.ProbeTypeLiveness)
cmdProbeConfig := &probes.CMDProbeConfig{
Command: args,
@ -283,10 +273,7 @@ func (r *Resources) probeBuilderLivenessCoreOperator(spec api.DeploymentSpec, gr
func (r *Resources) probeBuilderStartupCoreOperator(spec api.DeploymentSpec, group api.ServerGroup,
image api.ImageInfo) (Probe, error) {
args, err := r.probeCommand(spec, api.ProbeTypeStartUp)
if err != nil {
return nil, err
}
args := r.probeCommand(spec, api.ProbeTypeStartUp)
retries, periodSeconds := getProbeRetries(group)
if IsServerProgressAvailable(group, image) {
@ -399,10 +386,7 @@ func (r *Resources) probeBuilderReadinessCoreSelect() probeBuilder {
}
func (r *Resources) probeBuilderReadinessCoreOperator(spec api.DeploymentSpec, _ api.ServerGroup, _ api.ImageInfo) (Probe, error) {
args, err := r.probeCommand(spec, api.ProbeTypeReadiness)
if err != nil {
return nil, err
}
args := r.probeCommand(spec, api.ProbeTypeReadiness)
return &probes.CMDProbeConfig{
Command: args,

View file

@ -32,6 +32,7 @@ import (
"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
"github.com/arangodb/kube-arangodb/pkg/deployment/topology"
"github.com/arangodb/kube-arangodb/pkg/util"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
)
const (
@ -61,7 +62,7 @@ func containersCompare(ds api.DeploymentSpec, g api.ServerGroup, spec, status *c
g := podContainerFuncGenerator(ds, g, ac, bc)
if m, p, err := comparePodContainer(builder, g(compareServerContainerVolumeMounts)); err != nil {
if m, p, err := comparePodContainer(builder, g(compareServerContainerVolumeMounts), g(compareServerContainerProbes)); err != nil {
log.Err(err).Msg("Error while getting pod diff")
return SkippedRotation, nil, err
} else {
@ -104,11 +105,6 @@ func containersCompare(ds api.DeploymentSpec, g api.ServerGroup, spec, status *c
bc.Ports = ac.Ports
mode = mode.And(SilentRotation)
}
if !areProbesEqual(ac.StartupProbe, bc.StartupProbe) {
bc.StartupProbe = ac.StartupProbe
mode = mode.And(SilentRotation)
}
} else {
if ac.Image != bc.Image {
// Image changed
@ -252,6 +248,18 @@ func areProbesEqual(a, b *core.Probe) bool {
return equality.Semantic.DeepEqual(a, b)
}
func isManagedProbe(a, b *core.Probe) bool {
if a.Exec == nil || b.Exec == nil {
return false
}
if len(a.Exec.Command) == 0 || len(b.Exec.Command) == 0 {
return false
}
return a.Exec.Command[0] == k8sutil.LifecycleBinary() && b.Exec.Command[0] == k8sutil.LifecycleBinary()
}
// areEnvsFromEqual returns true when environment variables from source are the same after filtering.
func areEnvsFromEqual(a, b []core.EnvFromSource, rules ...func(a, b map[string]core.EnvFromSource) (map[string]core.EnvFromSource, map[string]core.EnvFromSource)) bool {
am := createEnvsFromMap(a)

View file

@ -0,0 +1,65 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
package rotation
import (
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/equality"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
)
func compareServerContainerProbes(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.Container) comparePodContainerFunc {
return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
if !areProbesEqual(spec.StartupProbe, status.StartupProbe) {
status.StartupProbe = spec.StartupProbe
mode = mode.And(SilentRotation)
}
if !areProbesEqual(spec.ReadinessProbe, status.ReadinessProbe) {
if isManagedProbe(spec.ReadinessProbe, status.ReadinessProbe) {
q := status.ReadinessProbe.DeepCopy()
q.Exec = spec.ReadinessProbe.Exec.DeepCopy()
if equality.Semantic.DeepDerivative(spec.ReadinessProbe, q) {
status.ReadinessProbe = spec.ReadinessProbe
mode = mode.And(SilentRotation)
}
}
}
if !areProbesEqual(spec.LivenessProbe, status.LivenessProbe) {
if isManagedProbe(spec.LivenessProbe, status.LivenessProbe) {
q := status.LivenessProbe.DeepCopy()
q.Exec = spec.LivenessProbe.Exec.DeepCopy()
if equality.Semantic.DeepDerivative(spec.LivenessProbe, q) {
status.LivenessProbe = spec.LivenessProbe
mode = mode.And(SilentRotation)
}
}
}
return
}
}

View file

@ -23,6 +23,7 @@ package rotation
import (
"encoding/json"
jd "github.com/josephburnett/jd/lib"
"github.com/rs/zerolog/log"
core "k8s.io/api/core/v1"
@ -114,14 +115,27 @@ func compare(deploymentSpec api.DeploymentSpec, member api.MemberStatus, group a
}
if spec.RotationNeeded(newStatus) {
specData, _ := json.Marshal(spec)
statusData, _ := json.Marshal(newStatus)
line := logger.Str("id", member.ID)
log.Info().Str("before", spec.PodSpecChecksum).
Str("id", member.ID).
Str("spec", string(specData)).
Str("status", string(statusData)).
Msg("Pod needs rotation - templates does not match")
specBytes, errA := json.Marshal(spec.PodSpec)
if errA == nil {
line = line.Str("spec", string(specBytes))
}
statusBytes, errB := json.Marshal(newStatus.PodSpec)
if errB == nil {
line = line.Str("status", string(statusBytes))
}
if errA == nil && errB == nil {
if specData, err := jd.ReadJsonString(string(specBytes)); err == nil && specData != nil {
if statusData, err := jd.ReadJsonString(string(statusBytes)); err == nil && statusData != nil {
line = line.Str("diff", specData.Diff(statusData).Render())
}
}
}
line.Info("Pod needs rotation - templates does not match")
return GracefulRotation, nil, nil
}

View file

@ -0,0 +1,27 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
package rotation
import "github.com/arangodb/kube-arangodb/pkg/logging"
var (
logger = logging.Global().RegisterAndGetLogger("pod_compare", logging.Info)
)

View file

@ -0,0 +1,76 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
package rotation
import (
_ "embed"
"encoding/json"
"testing"
"github.com/stretchr/testify/require"
core "k8s.io/api/core/v1"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
)
func init() {
k8sutil.SetBinaryPath("arangodb_operator")
}
//go:embed testdata/pod_lifecycle_change.000.spec.json
var podLifecycleChange000Spec []byte
//go:embed testdata/pod_lifecycle_change.000.status.json
var podLifecycleChange000Status []byte
func runPredefinedTests(t *testing.T, spec, status []byte) (mode Mode, plan api.Plan, err error) {
var specO, statusO core.PodTemplateSpec
require.NoError(t, json.Unmarshal(spec, &specO))
require.NoError(t, json.Unmarshal(status, &statusO))
specC, err := resources.ChecksumArangoPod(api.ServerGroupSpec{}, resources.CreatePodFromTemplate(&specO))
require.NoError(t, err)
statusC, err := resources.ChecksumArangoPod(api.ServerGroupSpec{}, resources.CreatePodFromTemplate(&statusO))
require.NoError(t, err)
obj := api.DeploymentSpec{}
member := api.MemberStatus{}
specT, err := api.GetArangoMemberPodTemplate(&specO, specC)
require.NoError(t, err)
statusT, err := api.GetArangoMemberPodTemplate(&statusO, statusC)
require.NoError(t, err)
return compare(obj, member, api.ServerGroupUnknown, specT, statusT)
}
func Test_PredefinedTests(t *testing.T) {
t.Run("podLifecycleChange000", func(t *testing.T) {
mode, plan, err := runPredefinedTests(t, podLifecycleChange000Spec, podLifecycleChange000Status)
require.NoError(t, err)
require.Empty(t, plan)
require.Equal(t, SilentRotation, mode)
})
}

View file

@ -0,0 +1,351 @@
{
"metadata": {
"creationTimestamp": null,
"finalizers": [
"pod.database.arangodb.com/delay",
"database.arangodb.com/graceful-shutdown"
],
"labels": {
"app": "arangodb",
"arango_deployment": "deployment",
"deployment.arangodb.com/member": "CRDN-0000000",
"role": "coordinator"
},
"name": "deployment-crdn-0000000-49237a"
},
"spec": {
"containers": [
{
"command": [
"/usr/sbin/arangod",
"--cluster.agency-endpoint=ssl://10.0.111.156:8529",
"--cluster.agency-endpoint=ssl://10.0.142.55:8529",
"--cluster.agency-endpoint=ssl://10.0.208.123:8529",
"--cluster.my-address=ssl://10.0.238.70:8529",
"--cluster.my-role=COORDINATOR",
"--database.directory=/data",
"--foxx.queues=true",
"--log.level=INFO",
"--log.output=+",
"--rocksdb.encryption-keyfile=/secrets/rocksdb/encryption/key",
"--server.authentication=true",
"--server.endpoint=ssl://[::]:8529",
"--server.jwt-secret-folder=/secrets/cluster/jwt",
"--server.statistics=true",
"--server.storage-engine=rocksdb",
"--ssl.ecdh-curve=",
"--ssl.keyfile=/secrets/tls/tls.keyfile",
"--log.thread=true",
"--log.thread-name=true",
"--server.authentication-system-only=false",
"--javascript.startup-options-denylist=.*",
"--javascript.endpoints-allowlist=example.com",
"--javascript.environment-variables-allowlist=^HOSTNAME$",
"--javascript.environment-variables-allowlist=^PATH$",
"--javascript.files-allowlist=^$",
"--javascript.harden",
"--server.harden",
"--javascript.files-allowlist=^(?!(\\/bin\/.*))(?!(\/dev\/.*))(?!(\/etc\/.*))(?!(\/lib\/.*))(?!(\/lifecycle\/.*))(?!(\/media\/.*))(?!(\/mnt\/.*))(?!(\/opt\/.*))(?!(\/proc\/.*))(?!(\/root\/.*))(?!(\/run\/.*))(?!(\/sbin\/.*))(?!(\/secrets\/.*))(?!(\/srv\/.*))(?!(\/usr\/.*))(?!(\/var\/.*)).+",
"--backup.api-enabled=jwt",
"--cluster.max-number-of-shards=1",
"--cluster.min-replication-factor=3",
"--cluster.max-replication-factor=3",
"--cluster.upgrade=online",
"--server.export-metrics-api=true",
"--cluster.force-one-shard=true"
],
"env": [
{
"name": "MY_POD_NAME",
"valueFrom": {
"fieldRef": {
"fieldPath": "metadata.name"
}
}
},
{
"name": "MY_POD_NAMESPACE",
"valueFrom": {
"fieldRef": {
"fieldPath": "metadata.namespace"
}
}
},
{
"name": "MY_NODE_NAME",
"valueFrom": {
"fieldRef": {
"fieldPath": "spec.nodeName"
}
}
},
{
"name": "NODE_NAME",
"valueFrom": {
"fieldRef": {
"fieldPath": "spec.nodeName"
}
}
},
{
"name": "ARANGODB_OVERRIDE_DETECTED_NUMBER_OF_CORES",
"value": "1"
},
{
"name": "ARANGODB_OVERRIDE_DETECTED_TOTAL_MEMORY",
"value": "1020054733"
},
{
"name": "ARANGODB_ZONE",
"value": "0"
},
{
"name": "ARANGODB_OVERRIDE_SERVER_GROUP",
"value": "coordinator"
},
{
"name": "ARANGODB_OVERRIDE_DEPLOYMENT_MODE",
"value": "Cluster"
},
{
"name": "ARANGODB_OVERRIDE_VERSION",
"value": "3.9.2"
},
{
"name": "ARANGODB_OVERRIDE_ENTERPRISE",
"value": "true"
}
],
"envFrom": [
{
"configMapRef": {
"name": "arangodb-operator-feature-config-map",
"optional": true
}
}
],
"image": "arangodb/arangodb",
"imagePullPolicy": "IfNotPresent",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/lifecycle/tools/arangodb_operator",
"lifecycle",
"preStop",
"port"
]
}
}
},
"name": "server",
"ports": [
{
"containerPort": 8529,
"name": "server",
"protocol": "TCP"
}
],
"readinessProbe": {
"exec": {
"command": [
"/lifecycle/tools/arangodb_operator",
"lifecycle",
"probe",
"readiness",
"--ssl",
"--auth"
]
},
"failureThreshold": 10,
"initialDelaySeconds": 2,
"periodSeconds": 2,
"successThreshold": 1,
"timeoutSeconds": 2
},
"resources": {
"limits": {
"cpu": "250m",
"memory": "1073741824"
},
"requests": {
"cpu": "250m",
"memory": "1073741824"
}
},
"securityContext": {
"capabilities": {
}
},
"startupProbe": {
"exec": {
"command": [
"/lifecycle/tools/arangodb_operator",
"lifecycle",
"probe",
"startup",
"--ssl",
"--auth"
]
},
"failureThreshold": 600,
"initialDelaySeconds": 1,
"periodSeconds": 5,
"successThreshold": 1,
"timeoutSeconds": 2
},
"volumeMounts": [
{
"mountPath": "/data",
"name": "arangod-data"
},
{
"mountPath": "/lifecycle/tools",
"name": "lifecycle"
},
{
"mountPath": "/secrets/tls",
"name": "tls-keyfile",
"readOnly": true
},
{
"mountPath": "/secrets/rocksdb/encryption",
"name": "rocksdb-encryption"
},
{
"mountPath": "/secrets/cluster/jwt",
"name": "cluster-jwt"
}
]
}
],
"hostname": "deployment-coordinator-0000000",
"initContainers": [
{
"command": [
"/usr/bin/arangodb_operator",
"lifecycle",
"copy",
"--target",
"/lifecycle/tools"
],
"image": "arangodb/kube-arangodb",
"imagePullPolicy": "IfNotPresent",
"name": "init-lifecycle",
"resources": {
},
"securityContext": {
"capabilities": {
}
},
"volumeMounts": [
{
"mountPath": "/lifecycle/tools",
"name": "lifecycle"
}
]
},
{
"command": [
"/usr/bin/arangodb_operator",
"uuid",
"--uuid-path",
"/data/UUID",
"--engine-path",
"/data/ENGINE",
"--uuid",
"CRDN-0000000",
"--engine",
"rocksdb"
],
"env": [
{
"name": "MY_POD_NAMESPACE",
"value": "depl-eayebbnrei1ao4ewwaac"
}
],
"image": "arangodb/kube-arangodb",
"name": "uuid",
"resources": {
"limits": {
"cpu": "100m",
"memory": "50Mi"
},
"requests": {
"cpu": "100m",
"memory": "10Mi"
}
},
"securityContext": {
"capabilities": {
}
},
"volumeMounts": [
{
"mountPath": "/data",
"name": "arangod-data"
}
]
}
],
"restartPolicy": "Never",
"serviceAccountName": "deployment-pod",
"subdomain": "deployment-int",
"terminationGracePeriodSeconds": 3600,
"tolerations": [
{
"effect": "NoSchedule",
"key": "node.arangodb.com/out-of-volumes",
"operator": "Exists"
},
{
"effect": "NoExecute",
"key": "node.kubernetes.io/not-ready",
"operator": "Exists",
"tolerationSeconds": 15
},
{
"effect": "NoExecute",
"key": "node.kubernetes.io/unreachable",
"operator": "Exists",
"tolerationSeconds": 15
},
{
"effect": "NoExecute",
"key": "node.alpha.kubernetes.io/unreachable",
"operator": "Exists",
"tolerationSeconds": 15
}
],
"volumes": [
{
"emptyDir": {
},
"name": "arangod-data"
},
{
"name": "tls-keyfile",
"secret": {
"secretName": "deployment-coordinator-0000000-tls-keyfile"
}
},
{
"name": "rocksdb-encryption",
"secret": {
"secretName": "deployment-rocksdb-encryption"
}
},
{
"name": "cluster-jwt",
"secret": {
"secretName": "deployment-jwt-folder"
}
},
{
"emptyDir": {
},
"name": "lifecycle"
}
]
}
}

View file

@ -0,0 +1,351 @@
{
"metadata": {
"creationTimestamp": null,
"finalizers": [
"pod.database.arangodb.com/delay",
"database.arangodb.com/graceful-shutdown"
],
"labels": {
"app": "arangodb",
"arango_deployment": "deployment",
"deployment.arangodb.com/member": "CRDN-0000000",
"role": "coordinator"
},
"name": "deployment-crdn-0000000-49237a"
},
"spec": {
"containers": [
{
"command": [
"/usr/sbin/arangod",
"--cluster.agency-endpoint=ssl://10.0.111.156:8529",
"--cluster.agency-endpoint=ssl://10.0.142.55:8529",
"--cluster.agency-endpoint=ssl://10.0.208.123:8529",
"--cluster.my-address=ssl://10.0.238.70:8529",
"--cluster.my-role=COORDINATOR",
"--database.directory=/data",
"--foxx.queues=true",
"--log.level=INFO",
"--log.output=+",
"--rocksdb.encryption-keyfile=/secrets/rocksdb/encryption/key",
"--server.authentication=true",
"--server.endpoint=ssl://[::]:8529",
"--server.jwt-secret-folder=/secrets/cluster/jwt",
"--server.statistics=true",
"--server.storage-engine=rocksdb",
"--ssl.ecdh-curve=",
"--ssl.keyfile=/secrets/tls/tls.keyfile",
"--log.thread=true",
"--log.thread-name=true",
"--server.authentication-system-only=false",
"--javascript.startup-options-denylist=.*",
"--javascript.endpoints-allowlist=example.com",
"--javascript.environment-variables-allowlist=^HOSTNAME$",
"--javascript.environment-variables-allowlist=^PATH$",
"--javascript.files-allowlist=^$",
"--javascript.harden",
"--server.harden",
"--javascript.files-allowlist=^(?!(\\/bin\/.*))(?!(\/dev\/.*))(?!(\/etc\/.*))(?!(\/lib\/.*))(?!(\/lifecycle\/.*))(?!(\/media\/.*))(?!(\/mnt\/.*))(?!(\/opt\/.*))(?!(\/proc\/.*))(?!(\/root\/.*))(?!(\/run\/.*))(?!(\/sbin\/.*))(?!(\/secrets\/.*))(?!(\/srv\/.*))(?!(\/usr\/.*))(?!(\/var\/.*)).+",
"--backup.api-enabled=jwt",
"--cluster.max-number-of-shards=1",
"--cluster.min-replication-factor=3",
"--cluster.max-replication-factor=3",
"--cluster.upgrade=online",
"--server.export-metrics-api=true",
"--cluster.force-one-shard=true"
],
"env": [
{
"name": "MY_POD_NAME",
"valueFrom": {
"fieldRef": {
"fieldPath": "metadata.name"
}
}
},
{
"name": "MY_POD_NAMESPACE",
"valueFrom": {
"fieldRef": {
"fieldPath": "metadata.namespace"
}
}
},
{
"name": "MY_NODE_NAME",
"valueFrom": {
"fieldRef": {
"fieldPath": "spec.nodeName"
}
}
},
{
"name": "NODE_NAME",
"valueFrom": {
"fieldRef": {
"fieldPath": "spec.nodeName"
}
}
},
{
"name": "ARANGODB_OVERRIDE_DETECTED_NUMBER_OF_CORES",
"value": "1"
},
{
"name": "ARANGODB_OVERRIDE_DETECTED_TOTAL_MEMORY",
"value": "1020054733"
},
{
"name": "ARANGODB_ZONE",
"value": "0"
},
{
"name": "ARANGODB_OVERRIDE_SERVER_GROUP",
"value": "coordinator"
},
{
"name": "ARANGODB_OVERRIDE_DEPLOYMENT_MODE",
"value": "Cluster"
},
{
"name": "ARANGODB_OVERRIDE_VERSION",
"value": "3.9.2"
},
{
"name": "ARANGODB_OVERRIDE_ENTERPRISE",
"value": "true"
}
],
"envFrom": [
{
"configMapRef": {
"name": "arangodb-operator-feature-config-map",
"optional": true
}
}
],
"image": "arangodb/arangodb",
"imagePullPolicy": "IfNotPresent",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/lifecycle/tools/arangodb_operator",
"lifecycle",
"preStop",
"port"
]
}
}
},
"name": "server",
"ports": [
{
"containerPort": 8529,
"name": "server",
"protocol": "TCP"
}
],
"readinessProbe": {
"exec": {
"command": [
"/lifecycle/tools/arangodb_operator",
"lifecycle",
"probe",
"--endpoint=/_admin/server/availability",
"--ssl",
"--auth"
]
},
"failureThreshold": 10,
"initialDelaySeconds": 2,
"periodSeconds": 2,
"successThreshold": 1,
"timeoutSeconds": 2
},
"resources": {
"limits": {
"cpu": "250m",
"memory": "1073741824"
},
"requests": {
"cpu": "250m",
"memory": "1073741824"
}
},
"securityContext": {
"capabilities": {
}
},
"startupProbe": {
"exec": {
"command": [
"/lifecycle/tools/arangodb_operator",
"lifecycle",
"probe",
"startup",
"--ssl",
"--auth"
]
},
"failureThreshold": 600,
"initialDelaySeconds": 1,
"periodSeconds": 5,
"successThreshold": 1,
"timeoutSeconds": 2
},
"volumeMounts": [
{
"mountPath": "/data",
"name": "arangod-data"
},
{
"mountPath": "/lifecycle/tools",
"name": "lifecycle"
},
{
"mountPath": "/secrets/tls",
"name": "tls-keyfile",
"readOnly": true
},
{
"mountPath": "/secrets/rocksdb/encryption",
"name": "rocksdb-encryption"
},
{
"mountPath": "/secrets/cluster/jwt",
"name": "cluster-jwt"
}
]
}
],
"hostname": "deployment-coordinator-0000000",
"initContainers": [
{
"command": [
"/usr/bin/arangodb_operator",
"lifecycle",
"copy",
"--target",
"/lifecycle/tools"
],
"image": "arangodb/kube-arangodb",
"imagePullPolicy": "IfNotPresent",
"name": "init-lifecycle",
"resources": {
},
"securityContext": {
"capabilities": {
}
},
"volumeMounts": [
{
"mountPath": "/lifecycle/tools",
"name": "lifecycle"
}
]
},
{
"command": [
"/usr/bin/arangodb_operator",
"uuid",
"--uuid-path",
"/data/UUID",
"--engine-path",
"/data/ENGINE",
"--uuid",
"CRDN-0000000",
"--engine",
"rocksdb"
],
"env": [
{
"name": "MY_POD_NAMESPACE",
"value": "depl-eayebbnrei1ao4ewwaac"
}
],
"image": "arangodb/kube-arangodb",
"name": "uuid",
"resources": {
"limits": {
"cpu": "100m",
"memory": "50Mi"
},
"requests": {
"cpu": "100m",
"memory": "10Mi"
}
},
"securityContext": {
"capabilities": {
}
},
"volumeMounts": [
{
"mountPath": "/data",
"name": "arangod-data"
}
]
}
],
"restartPolicy": "Never",
"serviceAccountName": "deployment-pod",
"subdomain": "deployment-int",
"terminationGracePeriodSeconds": 3600,
"tolerations": [
{
"effect": "NoSchedule",
"key": "node.arangodb.com/out-of-volumes",
"operator": "Exists"
},
{
"effect": "NoExecute",
"key": "node.kubernetes.io/not-ready",
"operator": "Exists",
"tolerationSeconds": 15
},
{
"effect": "NoExecute",
"key": "node.kubernetes.io/unreachable",
"operator": "Exists",
"tolerationSeconds": 15
},
{
"effect": "NoExecute",
"key": "node.alpha.kubernetes.io/unreachable",
"operator": "Exists",
"tolerationSeconds": 15
}
],
"volumes": [
{
"emptyDir": {
},
"name": "arangod-data"
},
{
"name": "tls-keyfile",
"secret": {
"secretName": "deployment-coordinator-0000000-tls-keyfile"
}
},
{
"name": "rocksdb-encryption",
"secret": {
"secretName": "deployment-rocksdb-encryption"
}
},
{
"name": "cluster-jwt",
"secret": {
"secretName": "deployment-jwt-folder"
}
},
{
"emptyDir": {
},
"name": "lifecycle"
}
]
}
}

View file

@ -36,6 +36,26 @@ const (
lifecycleVolumeName = "lifecycle"
)
var (
binaryPath string
)
func init() {
if b, err := os.Executable(); err != nil {
panic(err.Error())
} else {
binaryPath = b
}
}
func SetBinaryPath(path string) {
binaryPath = path
}
func LifecycleBinary() string {
return filepath.Join(LifecycleVolumeMountDir, filepath.Base(binaryPath))
}
// InitLifecycleContainer creates an init-container to copy the lifecycle binary to a shared volume.
func InitLifecycleContainer(image string, resources *core.ResourceRequirements, securityContext *core.SecurityContext) (core.Container, error) {
binaryPath, err := os.Executable()
@ -71,11 +91,7 @@ func NewLifecyclePort() (*core.Lifecycle, error) {
// NewLifecycle creates a lifecycle structure with preStop handler.
func NewLifecycle(t string) (*core.Lifecycle, error) {
binaryPath, err := os.Executable()
if err != nil {
return nil, errors.WithStack(err)
}
exePath := filepath.Join(LifecycleVolumeMountDir, filepath.Base(binaryPath))
exePath := LifecycleBinary()
lifecycle := &core.Lifecycle{
PreStop: &core.Handler{
Exec: &core.ExecAction{