From f62fdf2faa19ebfabcd39371d183426b3b177eef Mon Sep 17 00:00:00 2001 From: Adam Janikowski <12255597+ajanikow@users.noreply.github.com> Date: Thu, 26 Aug 2021 09:59:16 +0200 Subject: [PATCH] [Feature] ArangoMember template propagation (#773) --- CHANGELOG.md | 1 + main.go | 176 +++++++++++------- .../v1/arango_member_pod_template.go | 81 ++++++++ pkg/apis/deployment/v1/arango_member_spec.go | 4 +- .../deployment/v1/arango_member_status.go | 11 ++ pkg/apis/deployment/v1/conditions.go | 8 + pkg/apis/deployment/v1/deployment_status.go | 4 + pkg/apis/deployment/v1/plan.go | 4 + .../deployment/v1/zz_generated.deepcopy.go | 37 +++- .../v2alpha1/arango_member_pod_template.go | 81 ++++++++ .../deployment/v2alpha1/arango_member_spec.go | 4 +- .../v2alpha1/arango_member_status.go | 11 ++ pkg/apis/deployment/v2alpha1/conditions.go | 8 + .../deployment/v2alpha1/deployment_status.go | 4 + pkg/apis/deployment/v2alpha1/plan.go | 4 + .../v2alpha1/zz_generated.deepcopy.go | 37 +++- pkg/deployment/context_impl.go | 69 ++++++- pkg/deployment/deployment.go | 2 +- pkg/deployment/deployment_inspector.go | 8 +- pkg/deployment/deployment_run_test.go | 70 +++++-- pkg/deployment/deployment_suite_test.go | 11 ++ .../action_arango_membed_update_pod_spec.go | 152 +++++++++++++++ .../action_arango_membed_update_pod_status.go | 91 +++++++++ pkg/deployment/reconcile/action_context.go | 50 ++++- pkg/deployment/reconcile/context.go | 8 +- .../reconcile/plan_builder_appender.go | 95 ++++++++++ .../reconcile/plan_builder_context.go | 11 +- .../reconcile/plan_builder_factory.go | 94 ++++++++++ pkg/deployment/reconcile/plan_builder_high.go | 101 ++++++++-- .../reconcile/plan_builder_normal.go | 154 ++++++--------- .../reconcile/plan_builder_rotate_upgrade.go | 102 +++++----- .../reconcile/plan_builder_storage.go | 4 +- pkg/deployment/reconcile/plan_builder_test.go | 174 +++++++++++++---- pkg/deployment/reconcile/plan_builder_tls.go | 7 +- .../reconcile/plan_builder_tls_sni.go | 2 +- .../reconcile/plan_builder_update.go | 23 +++ .../reconcile/plan_builder_utils.go | 64 ------- pkg/deployment/resilience/member_failure.go | 2 +- pkg/deployment/resources/context.go | 36 +++- .../resources/inspector/inspector.go | 81 +++----- pkg/deployment/resources/pod_creator.go | 134 ++++++++----- pkg/deployment/resources/pod_inspector.go | 2 +- pkg/util/k8sutil/inspector/inspector.go | 6 +- 43 files changed, 1530 insertions(+), 498 deletions(-) create mode 100644 pkg/apis/deployment/v1/arango_member_pod_template.go create mode 100644 pkg/apis/deployment/v2alpha1/arango_member_pod_template.go create mode 100644 pkg/deployment/reconcile/action_arango_membed_update_pod_spec.go create mode 100644 pkg/deployment/reconcile/action_arango_membed_update_pod_status.go create mode 100644 pkg/deployment/reconcile/plan_builder_appender.go create mode 100644 pkg/deployment/reconcile/plan_builder_factory.go create mode 100644 pkg/deployment/reconcile/plan_builder_update.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8304c8b12..77d599da6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Add Pending Member phase - Add Ephemeral Volumes for apps feature - Check if the DB server is cleaned out. +- Render Pod Template in ArangoMember Spec and Status ## [1.2.1](https://github.com/arangodb/kube-arangodb/tree/1.2.1) (2021-07-28) - Fix ArangoMember race with multiple ArangoDeployments within single namespace diff --git a/main.go b/main.go index fcb7f8312..854b6af3f 100644 --- a/main.go +++ b/main.go @@ -28,11 +28,15 @@ import ( goflag "flag" "fmt" "net" + "net/http" "os" "strconv" "strings" "time" + operatorHTTP "github.com/arangodb/kube-arangodb/pkg/util/http" + "github.com/gin-gonic/gin" + "github.com/arangodb/kube-arangodb/pkg/version" "github.com/arangodb/kube-arangodb/pkg/util/arangod" @@ -110,6 +114,7 @@ var ( enableDeploymentReplication bool // Run deployment-replication operator enableStorage bool // Run local-storage operator enableBackup bool // Run backup operator + versionOnly bool // Run only version endpoint, explicitly disabled with other alpineImage, metricsExporterImage, arangoImage string @@ -143,6 +148,7 @@ func init() { f.BoolVar(&operatorOptions.enableDeploymentReplication, "operator.deployment-replication", false, "Enable to run the ArangoDeploymentReplication operator") f.BoolVar(&operatorOptions.enableStorage, "operator.storage", false, "Enable to run the ArangoLocalStorage operator") f.BoolVar(&operatorOptions.enableBackup, "operator.backup", false, "Enable to run the ArangoBackup operator") + f.BoolVar(&operatorOptions.versionOnly, "operator.version", false, "Enable only version endpoint in Operator") f.StringVar(&operatorOptions.alpineImage, "operator.alpine-image", UBIImageEnv.GetOrDefault(defaultAlpineImage), "Docker image used for alpine containers") f.MarkDeprecated("operator.alpine-image", "Value is not used anymore") f.StringVar(&operatorOptions.metricsExporterImage, "operator.metrics-exporter-image", MetricsExporterImageEnv.GetOrDefault(defaultMetricsExporterImage), "Docker image used for metrics containers by default") @@ -198,7 +204,11 @@ func cmdMainRun(cmd *cobra.Command, args []string) { // Check operating mode if !operatorOptions.enableDeployment && !operatorOptions.enableDeploymentReplication && !operatorOptions.enableStorage && !operatorOptions.enableBackup { - cliLog.Fatal().Err(err).Msg("Turn on --operator.deployment, --operator.deployment-replication, --operator.storage, --operator.backup or any combination of these") + if !operatorOptions.versionOnly { + cliLog.Fatal().Err(err).Msg("Turn on --operator.deployment, --operator.deployment-replication, --operator.storage, --operator.backup or any combination of these") + } + } else if operatorOptions.versionOnly { + cliLog.Fatal().Err(err).Msg("Options --operator.deployment, --operator.deployment-replication, --operator.storage, --operator.backup cannot be enabled together with --operator.version") } // Log version @@ -208,81 +218,111 @@ func cmdMainRun(cmd *cobra.Command, args []string) { Msgf("Starting arangodb-operator (%s), version %s build %s", version.GetVersionV1().Edition.Title(), version.GetVersionV1().Version, version.GetVersionV1().Build) // Check environment - if len(namespace) == 0 { - cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodNamespace) - } - if len(name) == 0 { - cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodName) - } - if len(ip) == 0 { - cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodIP) - } + if !operatorOptions.versionOnly { + if len(namespace) == 0 { + cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodNamespace) + } + if len(name) == 0 { + cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodName) + } + if len(ip) == 0 { + cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodIP) + } - // Get host name - id, err := os.Hostname() - if err != nil { - cliLog.Fatal().Err(err).Msg("Failed to get hostname") - } + // Get host name + id, err := os.Hostname() + if err != nil { + cliLog.Fatal().Err(err).Msg("Failed to get hostname") + } - // Create kubernetes client - kubecli, err := k8sutil.NewKubeClient() - if err != nil { - cliLog.Fatal().Err(err).Msg("Failed to create Kubernetes client") - } - secrets := kubecli.CoreV1().Secrets(namespace) + // Create kubernetes client + kubecli, err := k8sutil.NewKubeClient() + if err != nil { + cliLog.Fatal().Err(err).Msg("Failed to create Kubernetes client") + } + secrets := kubecli.CoreV1().Secrets(namespace) - // Create operator - cfg, deps, err := newOperatorConfigAndDeps(id+"-"+name, namespace, name) - if err != nil { - cliLog.Fatal().Err(err).Msg("Failed to create operator config & deps") - } - o, err := operator.NewOperator(cfg, deps) - if err != nil { - cliLog.Fatal().Err(err).Msg("Failed to create operator") - } + // Create operator + cfg, deps, err := newOperatorConfigAndDeps(id+"-"+name, namespace, name) + if err != nil { + cliLog.Fatal().Err(err).Msg("Failed to create operator config & deps") + } + o, err := operator.NewOperator(cfg, deps) + if err != nil { + cliLog.Fatal().Err(err).Msg("Failed to create operator") + } - listenAddr := net.JoinHostPort(serverOptions.host, strconv.Itoa(serverOptions.port)) - if svr, err := server.NewServer(kubecli.CoreV1(), server.Config{ - Namespace: namespace, - Address: listenAddr, - TLSSecretName: serverOptions.tlsSecretName, - TLSSecretNamespace: namespace, - PodName: name, - PodIP: ip, - AdminSecretName: serverOptions.adminSecretName, - AllowAnonymous: serverOptions.allowAnonymous, - }, server.Dependencies{ - Log: logService.MustGetLogger(logging.LoggerNameServer), - LivenessProbe: &livenessProbe, - Deployment: server.OperatorDependency{ - Enabled: cfg.EnableDeployment, - Probe: &deploymentProbe, - }, - DeploymentReplication: server.OperatorDependency{ - Enabled: cfg.EnableDeploymentReplication, - Probe: &deploymentReplicationProbe, - }, - Storage: server.OperatorDependency{ - Enabled: cfg.EnableStorage, - Probe: &storageProbe, - }, - Backup: server.OperatorDependency{ - Enabled: cfg.EnableBackup, - Probe: &backupProbe, - }, - Operators: o, + listenAddr := net.JoinHostPort(serverOptions.host, strconv.Itoa(serverOptions.port)) + if svr, err := server.NewServer(kubecli.CoreV1(), server.Config{ + Namespace: namespace, + Address: listenAddr, + TLSSecretName: serverOptions.tlsSecretName, + TLSSecretNamespace: namespace, + PodName: name, + PodIP: ip, + AdminSecretName: serverOptions.adminSecretName, + AllowAnonymous: serverOptions.allowAnonymous, + }, server.Dependencies{ + Log: logService.MustGetLogger(logging.LoggerNameServer), + LivenessProbe: &livenessProbe, + Deployment: server.OperatorDependency{ + Enabled: cfg.EnableDeployment, + Probe: &deploymentProbe, + }, + DeploymentReplication: server.OperatorDependency{ + Enabled: cfg.EnableDeploymentReplication, + Probe: &deploymentReplicationProbe, + }, + Storage: server.OperatorDependency{ + Enabled: cfg.EnableStorage, + Probe: &storageProbe, + }, + Backup: server.OperatorDependency{ + Enabled: cfg.EnableBackup, + Probe: &backupProbe, + }, + Operators: o, - Secrets: secrets, - }); err != nil { - cliLog.Fatal().Err(err).Msg("Failed to create HTTP server") + Secrets: secrets, + }); err != nil { + cliLog.Fatal().Err(err).Msg("Failed to create HTTP server") + } else { + go utilsError.LogError(cliLog, "error while starting service", svr.Run) + } + + // startChaos(context.Background(), cfg.KubeCli, cfg.Namespace, chaosLevel) + + // Start operator + o.Run() } else { - go utilsError.LogError(cliLog, "error while starting service", svr.Run) + if err := startVersionProcess(); err != nil { + cliLog.Fatal().Err(err).Msg("Failed to create HTTP server") + } + } +} + +func startVersionProcess() error { + // Just expose version + listenAddr := net.JoinHostPort(serverOptions.host, strconv.Itoa(serverOptions.port)) + cliLog.Info().Str("addr", listenAddr).Msgf("Starting version endpoint") + + gin.SetMode(gin.ReleaseMode) + r := gin.New() + r.Use(gin.Recovery()) + + versionV1Responser, err := operatorHTTP.NewSimpleJSONResponse(version.GetVersionV1()) + if err != nil { + return errors.WithStack(err) + } + r.GET("/_api/version", gin.WrapF(versionV1Responser.ServeHTTP)) + r.GET("/api/v1/version", gin.WrapF(versionV1Responser.ServeHTTP)) + + s := http.Server{ + Addr: listenAddr, + Handler: r, } - // startChaos(context.Background(), cfg.KubeCli, cfg.Namespace, chaosLevel) - - // Start operator - o.Run() + return s.ListenAndServe() } // newOperatorConfigAndDeps creates operator config & dependencies. diff --git a/pkg/apis/deployment/v1/arango_member_pod_template.go b/pkg/apis/deployment/v1/arango_member_pod_template.go new file mode 100644 index 000000000..2c18b4f8e --- /dev/null +++ b/pkg/apis/deployment/v1/arango_member_pod_template.go @@ -0,0 +1,81 @@ +// +// DISCLAIMER +// +// Copyright 2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package v1 + +import ( + "crypto/sha256" + "fmt" + + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/json" +) + +func GetArangoMemberPodTemplate(pod *core.PodTemplateSpec, podSpecChecksum string) (*ArangoMemberPodTemplate, error) { + data, err := json.Marshal(pod.Spec) + if err != nil { + return nil, err + } + + return &ArangoMemberPodTemplate{ + PodSpec: pod, + PodSpecChecksum: podSpecChecksum, + Checksum: fmt.Sprintf("%0x", sha256.Sum256(data)), + }, nil +} + +type ArangoMemberPodTemplate struct { + PodSpec *core.PodTemplateSpec `json:"podSpec,omitempty"` + PodSpecChecksum string `json:"podSpecChecksum,omitempty"` + Checksum string `json:"checksum,omitempty"` +} + +func (a *ArangoMemberPodTemplate) Equals(b *ArangoMemberPodTemplate) bool { + if a == nil && b == nil { + return true + } + + if a == nil || b == nil { + return false + } + + return a.Checksum == b.Checksum && a.PodSpecChecksum == b.PodSpecChecksum +} + +func (a *ArangoMemberPodTemplate) RotationNeeded(b *ArangoMemberPodTemplate) bool { + if a == nil && b == nil { + return true + } + + if a == nil || b == nil { + return true + } + + return a.PodSpecChecksum != b.PodSpecChecksum +} + +func (a *ArangoMemberPodTemplate) EqualPodSpecChecksum(checksum string) bool { + if a == nil { + return false + } + return checksum == a.PodSpecChecksum +} diff --git a/pkg/apis/deployment/v1/arango_member_spec.go b/pkg/apis/deployment/v1/arango_member_spec.go index 0bf5bbfec..073f47634 100644 --- a/pkg/apis/deployment/v1/arango_member_spec.go +++ b/pkg/apis/deployment/v1/arango_member_spec.go @@ -23,7 +23,6 @@ package v1 import ( - core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" ) @@ -32,6 +31,5 @@ type ArangoMemberSpec struct { ID string `json:"id,omitempty"` DeploymentUID types.UID `json:"deploymentUID,omitempty"` - Template *core.PodTemplate `json:"template,omitempty"` - TemplateChecksum string `json:"templateChecksum,omitempty"` + Template *ArangoMemberPodTemplate `json:"template,omitempty"` } diff --git a/pkg/apis/deployment/v1/arango_member_status.go b/pkg/apis/deployment/v1/arango_member_status.go index 00d1a9c58..2b9145c60 100644 --- a/pkg/apis/deployment/v1/arango_member_status.go +++ b/pkg/apis/deployment/v1/arango_member_status.go @@ -22,5 +22,16 @@ package v1 +const ( + ArangoMemberConditionPendingRestart ConditionType = "pending-restart" +) + type ArangoMemberStatus struct { + Conditions ConditionList `json:"conditions,omitempty"` + + Template *ArangoMemberPodTemplate `json:"template,omitempty"` +} + +func (a ArangoMemberStatus) IsPendingRestart() bool { + return a.Conditions.IsTrue(ArangoMemberConditionPendingRestart) } diff --git a/pkg/apis/deployment/v1/conditions.go b/pkg/apis/deployment/v1/conditions.go index 92f5a5cd6..a02bf6c2f 100644 --- a/pkg/apis/deployment/v1/conditions.go +++ b/pkg/apis/deployment/v1/conditions.go @@ -31,6 +31,10 @@ import ( // ConditionType is a strongly typed condition name type ConditionType string +func (c ConditionType) String() string { + return string(c) +} + const ( // ConditionTypeReady indicates that the member or entire deployment is ready and running normally. ConditionTypeReady ConditionType = "Ready" @@ -67,6 +71,10 @@ const ( ConditionTypeUpgradeFailed ConditionType = "UpgradeFailed" // ConditionTypeMaintenanceMode indicates that Maintenance is enabled ConditionTypeMaintenanceMode ConditionType = "MaintenanceMode" + // ConditionTypePendingRestart indicates that restart is required + ConditionTypePendingRestart ConditionType = "PendingRestart" + // ConditionTypePendingTLSRotation indicates that TLS rotation is pending + ConditionTypePendingTLSRotation ConditionType = "PendingTLSRotation" ) // Condition represents one current condition of a deployment or deployment member. diff --git a/pkg/apis/deployment/v1/deployment_status.go b/pkg/apis/deployment/v1/deployment_status.go index 8fe7f125f..3430a0e96 100644 --- a/pkg/apis/deployment/v1/deployment_status.go +++ b/pkg/apis/deployment/v1/deployment_status.go @@ -102,3 +102,7 @@ func (ds *DeploymentStatus) Equal(other DeploymentStatus) bool { func (ds *DeploymentStatus) IsForceReload() bool { return util.BoolOrDefault(ds.ForceStatusReload, false) } + +func (ds *DeploymentStatus) IsPlanEmpty() bool { + return ds.Plan.IsEmpty() && ds.HighPriorityPlan.IsEmpty() +} diff --git a/pkg/apis/deployment/v1/plan.go b/pkg/apis/deployment/v1/plan.go index 6fee80238..36c71f532 100644 --- a/pkg/apis/deployment/v1/plan.go +++ b/pkg/apis/deployment/v1/plan.go @@ -161,6 +161,10 @@ const ( ActionTypeSetMemberCondition ActionType = "SetMemberCondition" // ActionTypeMemberRIDUpdate updated member Run ID (UID). High priority ActionTypeMemberRIDUpdate ActionType = "MemberRIDUpdate" + // ActionTypeArangoMemberUpdatePodSpec updates pod spec + ActionTypeArangoMemberUpdatePodSpec ActionType = "ArangoMemberUpdatePodSpec" + // ActionTypeArangoMemberUpdatePodStatus updates pod spec + ActionTypeArangoMemberUpdatePodStatus ActionType = "ArangoMemberUpdatePodStatus" ) const ( diff --git a/pkg/apis/deployment/v1/zz_generated.deepcopy.go b/pkg/apis/deployment/v1/zz_generated.deepcopy.go index 3ea5dedcc..837c21381 100644 --- a/pkg/apis/deployment/v1/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1/zz_generated.deepcopy.go @@ -149,7 +149,7 @@ func (in *ArangoMember) DeepCopyInto(out *ArangoMember) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) return } @@ -204,12 +204,33 @@ func (in *ArangoMemberList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArangoMemberPodTemplate) DeepCopyInto(out *ArangoMemberPodTemplate) { + *out = *in + if in.PodSpec != nil { + in, out := &in.PodSpec, &out.PodSpec + *out = new(corev1.PodTemplateSpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMemberPodTemplate. +func (in *ArangoMemberPodTemplate) DeepCopy() *ArangoMemberPodTemplate { + if in == nil { + return nil + } + out := new(ArangoMemberPodTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ArangoMemberSpec) DeepCopyInto(out *ArangoMemberSpec) { *out = *in if in.Template != nil { in, out := &in.Template, &out.Template - *out = new(corev1.PodTemplate) + *out = new(ArangoMemberPodTemplate) (*in).DeepCopyInto(*out) } return @@ -228,6 +249,18 @@ func (in *ArangoMemberSpec) DeepCopy() *ArangoMemberSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ArangoMemberStatus) DeepCopyInto(out *ArangoMemberStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(ConditionList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Template != nil { + in, out := &in.Template, &out.Template + *out = new(ArangoMemberPodTemplate) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/apis/deployment/v2alpha1/arango_member_pod_template.go b/pkg/apis/deployment/v2alpha1/arango_member_pod_template.go new file mode 100644 index 000000000..15b203b9a --- /dev/null +++ b/pkg/apis/deployment/v2alpha1/arango_member_pod_template.go @@ -0,0 +1,81 @@ +// +// DISCLAIMER +// +// Copyright 2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package v2alpha1 + +import ( + "crypto/sha256" + "fmt" + + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/json" +) + +func GetArangoMemberPodTemplate(pod *core.PodTemplateSpec, podSpecChecksum string) (*ArangoMemberPodTemplate, error) { + data, err := json.Marshal(pod.Spec) + if err != nil { + return nil, err + } + + return &ArangoMemberPodTemplate{ + PodSpec: pod, + PodSpecChecksum: podSpecChecksum, + Checksum: fmt.Sprintf("%0x", sha256.Sum256(data)), + }, nil +} + +type ArangoMemberPodTemplate struct { + PodSpec *core.PodTemplateSpec `json:"podSpec,omitempty"` + PodSpecChecksum string `json:"podSpecChecksum,omitempty"` + Checksum string `json:"checksum,omitempty"` +} + +func (a *ArangoMemberPodTemplate) Equals(b *ArangoMemberPodTemplate) bool { + if a == nil && b == nil { + return true + } + + if a == nil || b == nil { + return false + } + + return a.Checksum == b.Checksum && a.PodSpecChecksum == b.PodSpecChecksum +} + +func (a *ArangoMemberPodTemplate) RotationNeeded(b *ArangoMemberPodTemplate) bool { + if a == nil && b == nil { + return true + } + + if a == nil || b == nil { + return true + } + + return a.PodSpecChecksum != b.PodSpecChecksum +} + +func (a *ArangoMemberPodTemplate) EqualPodSpecChecksum(checksum string) bool { + if a == nil { + return false + } + return checksum == a.PodSpecChecksum +} diff --git a/pkg/apis/deployment/v2alpha1/arango_member_spec.go b/pkg/apis/deployment/v2alpha1/arango_member_spec.go index ab4af9bc4..a9bf77c63 100644 --- a/pkg/apis/deployment/v2alpha1/arango_member_spec.go +++ b/pkg/apis/deployment/v2alpha1/arango_member_spec.go @@ -23,7 +23,6 @@ package v2alpha1 import ( - core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" ) @@ -32,6 +31,5 @@ type ArangoMemberSpec struct { ID string `json:"id,omitempty"` DeploymentUID types.UID `json:"deploymentUID,omitempty"` - Template *core.PodTemplate `json:"template,omitempty"` - TemplateChecksum string `json:"templateChecksum,omitempty"` + Template *ArangoMemberPodTemplate `json:"template,omitempty"` } diff --git a/pkg/apis/deployment/v2alpha1/arango_member_status.go b/pkg/apis/deployment/v2alpha1/arango_member_status.go index e06a1e93b..21a5b8540 100644 --- a/pkg/apis/deployment/v2alpha1/arango_member_status.go +++ b/pkg/apis/deployment/v2alpha1/arango_member_status.go @@ -22,5 +22,16 @@ package v2alpha1 +const ( + ArangoMemberConditionPendingRestart ConditionType = "pending-restart" +) + type ArangoMemberStatus struct { + Conditions ConditionList `json:"conditions,omitempty"` + + Template *ArangoMemberPodTemplate `json:"template,omitempty"` +} + +func (a ArangoMemberStatus) IsPendingRestart() bool { + return a.Conditions.IsTrue(ArangoMemberConditionPendingRestart) } diff --git a/pkg/apis/deployment/v2alpha1/conditions.go b/pkg/apis/deployment/v2alpha1/conditions.go index 52baa6ab8..0dbd791c4 100644 --- a/pkg/apis/deployment/v2alpha1/conditions.go +++ b/pkg/apis/deployment/v2alpha1/conditions.go @@ -31,6 +31,10 @@ import ( // ConditionType is a strongly typed condition name type ConditionType string +func (c ConditionType) String() string { + return string(c) +} + const ( // ConditionTypeReady indicates that the member or entire deployment is ready and running normally. ConditionTypeReady ConditionType = "Ready" @@ -67,6 +71,10 @@ const ( ConditionTypeUpgradeFailed ConditionType = "UpgradeFailed" // ConditionTypeMaintenanceMode indicates that Maintenance is enabled ConditionTypeMaintenanceMode ConditionType = "MaintenanceMode" + // ConditionTypePendingRestart indicates that restart is required + ConditionTypePendingRestart ConditionType = "PendingRestart" + // ConditionTypePendingTLSRotation indicates that TLS rotation is pending + ConditionTypePendingTLSRotation ConditionType = "PendingTLSRotation" ) // Condition represents one current condition of a deployment or deployment member. diff --git a/pkg/apis/deployment/v2alpha1/deployment_status.go b/pkg/apis/deployment/v2alpha1/deployment_status.go index 66ad15a8a..2185064cc 100644 --- a/pkg/apis/deployment/v2alpha1/deployment_status.go +++ b/pkg/apis/deployment/v2alpha1/deployment_status.go @@ -102,3 +102,7 @@ func (ds *DeploymentStatus) Equal(other DeploymentStatus) bool { func (ds *DeploymentStatus) IsForceReload() bool { return util.BoolOrDefault(ds.ForceStatusReload, false) } + +func (ds *DeploymentStatus) IsPlanEmpty() bool { + return ds.Plan.IsEmpty() && ds.HighPriorityPlan.IsEmpty() +} diff --git a/pkg/apis/deployment/v2alpha1/plan.go b/pkg/apis/deployment/v2alpha1/plan.go index 095da9f36..961bc4fb5 100644 --- a/pkg/apis/deployment/v2alpha1/plan.go +++ b/pkg/apis/deployment/v2alpha1/plan.go @@ -161,6 +161,10 @@ const ( ActionTypeSetMemberCondition ActionType = "SetMemberCondition" // ActionTypeMemberRIDUpdate updated member Run ID (UID). High priority ActionTypeMemberRIDUpdate ActionType = "MemberRIDUpdate" + // ActionTypeArangoMemberUpdatePodSpec updates pod spec + ActionTypeArangoMemberUpdatePodSpec ActionType = "ArangoMemberUpdatePodSpec" + // ActionTypeArangoMemberUpdatePodStatus updates pod spec + ActionTypeArangoMemberUpdatePodStatus ActionType = "ArangoMemberUpdatePodStatus" ) const ( diff --git a/pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go b/pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go index a7aa656f1..200866a92 100644 --- a/pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go @@ -149,7 +149,7 @@ func (in *ArangoMember) DeepCopyInto(out *ArangoMember) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) return } @@ -204,12 +204,33 @@ func (in *ArangoMemberList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArangoMemberPodTemplate) DeepCopyInto(out *ArangoMemberPodTemplate) { + *out = *in + if in.PodSpec != nil { + in, out := &in.PodSpec, &out.PodSpec + *out = new(v1.PodTemplateSpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMemberPodTemplate. +func (in *ArangoMemberPodTemplate) DeepCopy() *ArangoMemberPodTemplate { + if in == nil { + return nil + } + out := new(ArangoMemberPodTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ArangoMemberSpec) DeepCopyInto(out *ArangoMemberSpec) { *out = *in if in.Template != nil { in, out := &in.Template, &out.Template - *out = new(v1.PodTemplate) + *out = new(ArangoMemberPodTemplate) (*in).DeepCopyInto(*out) } return @@ -228,6 +249,18 @@ func (in *ArangoMemberSpec) DeepCopy() *ArangoMemberSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ArangoMemberStatus) DeepCopyInto(out *ArangoMemberStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(ConditionList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Template != nil { + in, out := &in.Template, &out.Template + *out = new(ArangoMemberPodTemplate) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/deployment/context_impl.go b/pkg/deployment/context_impl.go index f6335ce83..bfe40394e 100644 --- a/pkg/deployment/context_impl.go +++ b/pkg/deployment/context_impl.go @@ -64,7 +64,7 @@ import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/resources" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" - v1 "k8s.io/api/core/v1" + core "k8s.io/api/core/v1" ) var _ resources.Context = &Deployment{} @@ -381,7 +381,7 @@ func (d *Deployment) CreateMember(ctx context.Context, group api.ServerGroup, id } // GetPod returns pod. -func (d *Deployment) GetPod(ctx context.Context, podName string) (*v1.Pod, error) { +func (d *Deployment) GetPod(ctx context.Context, podName string) (*core.Pod, error) { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) defer cancel() @@ -405,7 +405,7 @@ func (d *Deployment) DeletePod(ctx context.Context, podName string) error { // CleanupPod deletes a given pod with force and explicit UID. // If the pod does not exist, the error is ignored. -func (d *Deployment) CleanupPod(ctx context.Context, p *v1.Pod) error { +func (d *Deployment) CleanupPod(ctx context.Context, p *core.Pod) error { log := d.deps.Log podName := p.GetName() ns := p.GetNamespace() @@ -462,7 +462,7 @@ func (d *Deployment) DeletePvc(ctx context.Context, pvcName string) error { // UpdatePvc updated a persistent volume claim in the namespace // of the deployment. If the pvc does not exist, the error is ignored. -func (d *Deployment) UpdatePvc(ctx context.Context, pvc *v1.PersistentVolumeClaim) error { +func (d *Deployment) UpdatePvc(ctx context.Context, pvc *core.PersistentVolumeClaim) error { err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { _, err := d.GetKubeCli().CoreV1().PersistentVolumeClaims(d.GetNamespace()).Update(ctxChild, pvc, meta.UpdateOptions{}) return err @@ -479,7 +479,7 @@ func (d *Deployment) UpdatePvc(ctx context.Context, pvc *v1.PersistentVolumeClai } // GetOwnedPVCs returns a list of all PVCs owned by the deployment. -func (d *Deployment) GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) { +func (d *Deployment) GetOwnedPVCs() ([]core.PersistentVolumeClaim, error) { // Get all current PVCs log := d.deps.Log pvcs, err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(d.GetNamespace()).List(context.Background(), k8sutil.DeploymentListOpt(d.GetName())) @@ -487,7 +487,7 @@ func (d *Deployment) GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) { log.Debug().Err(err).Msg("Failed to list PVCs") return nil, errors.WithStack(err) } - myPVCs := make([]v1.PersistentVolumeClaim, 0, len(pvcs.Items)) + myPVCs := make([]core.PersistentVolumeClaim, 0, len(pvcs.Items)) for _, p := range pvcs.Items { if d.isOwnerOf(&p) { myPVCs = append(myPVCs, p) @@ -497,7 +497,7 @@ func (d *Deployment) GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) { } // GetPvc gets a PVC by the given name, in the samespace of the deployment. -func (d *Deployment) GetPvc(ctx context.Context, pvcName string) (*v1.PersistentVolumeClaim, error) { +func (d *Deployment) GetPvc(ctx context.Context, pvcName string) (*core.PersistentVolumeClaim, error) { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) defer cancel() @@ -577,14 +577,30 @@ func (d *Deployment) GetAgencyData(ctx context.Context, i interface{}, keyParts return err } -func (d *Deployment) RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*v1.Pod, error) { +func (d *Deployment) RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) { return d.resources.RenderPodForMember(ctx, cachedStatus, spec, status, memberID, imageInfo) } +func (d *Deployment) RenderPodForMemberFromCurrent(ctx context.Context, cachedStatus inspectorInterface.Inspector, memberID string) (*core.Pod, error) { + return d.resources.RenderPodForMemberFromCurrent(ctx, cachedStatus, memberID) +} + +func (d *Deployment) RenderPodTemplateForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.PodTemplateSpec, error) { + return d.resources.RenderPodTemplateForMember(ctx, cachedStatus, spec, status, memberID, imageInfo) +} + +func (d *Deployment) RenderPodTemplateForMemberFromCurrent(ctx context.Context, cachedStatus inspectorInterface.Inspector, memberID string) (*core.PodTemplateSpec, error) { + return d.resources.RenderPodTemplateForMemberFromCurrent(ctx, cachedStatus, memberID) +} + func (d *Deployment) SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) { return d.resources.SelectImage(spec, status) } +func (d *Deployment) SelectImageForMember(spec api.DeploymentSpec, status api.DeploymentStatus, member api.MemberStatus) (api.ImageInfo, bool) { + return d.resources.SelectImageForMember(spec, status, member) +} + func (d *Deployment) GetMetricsExporterImage() string { return d.config.MetricsExporterImage } @@ -616,7 +632,7 @@ func (d *Deployment) GetName() string { return d.name } -func (d *Deployment) GetOwnedPods(ctx context.Context) ([]v1.Pod, error) { +func (d *Deployment) GetOwnedPods(ctx context.Context) ([]core.Pod, error) { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) defer cancel() @@ -625,7 +641,7 @@ func (d *Deployment) GetOwnedPods(ctx context.Context) ([]v1.Pod, error) { return nil, err } - podList := make([]v1.Pod, 0, len(pods.Items)) + podList := make([]core.Pod, 0, len(pods.Items)) for _, p := range pods.Items { if !d.isOwnerOf(&p) { @@ -645,3 +661,36 @@ func (d *Deployment) GetCachedStatus() inspectorInterface.Inspector { func (d *Deployment) SetCachedStatus(i inspectorInterface.Inspector) { d.currentState = i } + +func (d *Deployment) WithArangoMemberUpdate(ctx context.Context, namespace, name string, action resources.ArangoMemberUpdateFunc) error { + o, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoMembers(namespace).Get(ctx, name, meta.GetOptions{}) + if err != nil { + return err + } + + if action(o) { + if _, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoMembers(namespace).Update(ctx, o, meta.UpdateOptions{}); err != nil { + return err + } + } + + return nil +} + +func (d *Deployment) WithArangoMemberStatusUpdate(ctx context.Context, namespace, name string, action resources.ArangoMemberStatusUpdateFunc) error { + o, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoMembers(namespace).Get(ctx, name, meta.GetOptions{}) + if err != nil { + return err + } + + status := o.Status.DeepCopy() + + if action(o, status) { + o.Status = *status + if _, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoMembers(namespace).UpdateStatus(ctx, o, meta.UpdateOptions{}); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index 59f776e0d..57245dd5c 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -298,7 +298,7 @@ func (d *Deployment) run() { for { select { case <-d.stopCh: - cachedStatus, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetArangoCli(), d.GetNamespace()) + cachedStatus, err := inspector.NewInspector(context.Background(), d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetArangoCli(), d.GetNamespace()) if err != nil { log.Error().Err(err).Msg("Unable to get resources") } diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index 0748483a7..884b2b5d3 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -104,7 +104,7 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval deploymentName := d.GetName() defer metrics.SetDuration(inspectDeploymentDurationGauges.WithLabelValues(deploymentName), start) - cachedStatus, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetArangoCli(), d.GetNamespace()) + cachedStatus, err := inspector.NewInspector(context.Background(), d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetArangoCli(), d.GetNamespace()) if err != nil { log.Error().Err(err).Msg("Unable to get resources") return minInspectionInterval // Retry ASAP @@ -293,7 +293,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva return minInspectionInterval, nil } - if d.apiObject.Status.Plan.IsEmpty() && status.AppliedVersion != checksum { + if d.apiObject.Status.IsPlanEmpty() && status.AppliedVersion != checksum { if err := d.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { s.AppliedVersion = checksum return true @@ -303,7 +303,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva return minInspectionInterval, nil } else if status.AppliedVersion == checksum { - if !status.Plan.IsEmpty() && status.Conditions.IsTrue(api.ConditionTypeUpToDate) { + if !d.apiObject.Status.IsPlanEmpty() && status.Conditions.IsTrue(api.ConditionTypeUpToDate) { if err = d.updateCondition(ctx, api.ConditionTypeUpToDate, false, "Plan is not empty", "There are pending operations in plan"); err != nil { return minInspectionInterval, errors.Wrapf(err, "Unable to update UpToDate condition") } @@ -311,7 +311,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva return minInspectionInterval, nil } - if status.Plan.IsEmpty() && !status.Conditions.IsTrue(api.ConditionTypeUpToDate) { + if d.apiObject.Status.IsPlanEmpty() && !status.Conditions.IsTrue(api.ConditionTypeUpToDate) { if err = d.updateCondition(ctx, api.ConditionTypeUpToDate, true, "Spec is Up To Date", "Spec is Up To Date"); err != nil { return minInspectionInterval, errors.Wrapf(err, "Unable to update UpToDate condition") } diff --git a/pkg/deployment/deployment_run_test.go b/pkg/deployment/deployment_run_test.go index fb17402f7..28d37c140 100644 --- a/pkg/deployment/deployment_run_test.go +++ b/pkg/deployment/deployment_run_test.go @@ -29,9 +29,10 @@ import ( "fmt" "testing" - "github.com/arangodb/kube-arangodb/pkg/util/errors" - "github.com/arangodb/kube-arangodb/pkg/deployment/features" + "github.com/arangodb/kube-arangodb/pkg/deployment/resources" + + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/rs/zerolog/log" @@ -65,7 +66,7 @@ func runTestCase(t *testing.T, testCase testCaseStruct) { errs := 0 for { - cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetArangoCli(), d.GetNamespace()) + cache, err := inspector.NewInspector(context.Background(), d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetArangoCli(), d.GetNamespace()) require.NoError(t, err) err = d.resources.EnsureSecrets(context.Background(), log.Logger, cache) if err == nil { @@ -96,6 +97,16 @@ func runTestCase(t *testing.T, testCase testCaseStruct) { if testCase.Resources != nil { testCase.Resources(t, d) } + + // Set features + { + *features.EncryptionRotation().EnabledPointer() = testCase.Features.EncryptionRotation + require.Equal(t, testCase.Features.EncryptionRotation, *features.EncryptionRotation().EnabledPointer()) + *features.JWTRotation().EnabledPointer() = testCase.Features.JWTRotation + *features.TLSSNI().EnabledPointer() = testCase.Features.TLSSNI + *features.TLSRotation().EnabledPointer() = testCase.Features.TLSRotation + } + // Set Pending phase require.NoError(t, d.status.last.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { for _, m := range list { @@ -110,8 +121,11 @@ func runTestCase(t *testing.T, testCase testCaseStruct) { })) // Set members - require.NoError(t, d.status.last.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { + if err := d.status.last.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { for _, m := range list { + c := d.GetArangoCli() + k := d.GetKubeCli() + member := api.ArangoMember{ ObjectMeta: metav1.ObjectMeta{ Namespace: d.GetNamespace(), @@ -123,7 +137,6 @@ func runTestCase(t *testing.T, testCase testCaseStruct) { }, } - c := d.GetArangoCli() if _, err := c.DatabaseV1().ArangoMembers(member.GetNamespace()).Create(context.Background(), &member, metav1.CreateOptions{}); err != nil { return err } @@ -135,26 +148,51 @@ func runTestCase(t *testing.T, testCase testCaseStruct) { }, } - k := d.GetKubeCli() if _, err := k.CoreV1().Services(member.GetNamespace()).Create(context.Background(), &s, metav1.CreateOptions{}); err != nil { return err } + + cache, err := inspector.NewInspector(context.Background(), d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetArangoCli(), d.GetNamespace()) + require.NoError(t, err) + + groupSpec := d.apiObject.Spec.GetServerGroupSpec(group) + + image, ok := d.resources.SelectImage(d.apiObject.Spec, d.status.last) + require.True(t, ok) + + template, err := d.resources.RenderPodTemplateForMember(context.Background(), cache, d.apiObject.Spec, d.status.last, m.ID, image) + if err != nil { + return err + } + + checksum, err := resources.ChecksumArangoPod(groupSpec, resources.CreatePodFromTemplate(template)) + require.NoError(t, err) + + podTemplate, err := api.GetArangoMemberPodTemplate(template, checksum) + require.NoError(t, err) + + member.Status.Template = podTemplate + member.Spec.Template = podTemplate + + if _, err := c.DatabaseV1().ArangoMembers(member.GetNamespace()).Update(context.Background(), &member, metav1.UpdateOptions{}); err != nil { + return err + } + + if _, err := c.DatabaseV1().ArangoMembers(member.GetNamespace()).UpdateStatus(context.Background(), &member, metav1.UpdateOptions{}); err != nil { + return err + } } return nil - })) - - // Set features - { - *features.EncryptionRotation().EnabledPointer() = testCase.Features.EncryptionRotation - require.Equal(t, testCase.Features.EncryptionRotation, *features.EncryptionRotation().EnabledPointer()) - *features.JWTRotation().EnabledPointer() = testCase.Features.JWTRotation - *features.TLSSNI().EnabledPointer() = testCase.Features.TLSSNI - *features.TLSRotation().EnabledPointer() = testCase.Features.TLSRotation + }); err != nil { + if testCase.ExpectedError != nil && assert.EqualError(t, err, testCase.ExpectedError.Error()) { + return + } + require.NoError(t, err) } // Act - cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetArangoCli(), d.GetNamespace()) + cache, err := inspector.NewInspector(context.Background(), d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetArangoCli(), d.GetNamespace()) require.NoError(t, err) err = d.resources.EnsurePods(context.Background(), cache) diff --git a/pkg/deployment/deployment_suite_test.go b/pkg/deployment/deployment_suite_test.go index 7ded8b599..1cfbea446 100644 --- a/pkg/deployment/deployment_suite_test.go +++ b/pkg/deployment/deployment_suite_test.go @@ -455,6 +455,17 @@ func createTestDeployment(config Config, arangoDeployment *api.ArangoDeployment) Namespace: testNamespace, } + arangoDeployment.Status.Images = api.ImageInfoList{ + { + Image: "arangodb/arangodb:latest", + ImageID: "arangodb/arangodb:latest", + ArangoDBVersion: "1.0.0", + Enterprise: false, + }, + } + + arangoDeployment.Status.CurrentImage = &arangoDeployment.Status.Images[0] + deps := Dependencies{ Log: zerolog.New(ioutil.Discard), KubeCli: kubernetesClientSet, diff --git a/pkg/deployment/reconcile/action_arango_membed_update_pod_spec.go b/pkg/deployment/reconcile/action_arango_membed_update_pod_spec.go new file mode 100644 index 000000000..121663cd0 --- /dev/null +++ b/pkg/deployment/reconcile/action_arango_membed_update_pod_spec.go @@ -0,0 +1,152 @@ +// +// DISCLAIMER +// +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + + "github.com/arangodb/kube-arangodb/pkg/deployment/resources" + "github.com/arangodb/kube-arangodb/pkg/util/errors" + "github.com/rs/zerolog/log" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/rs/zerolog" +) + +func init() { + registerAction(api.ActionTypeArangoMemberUpdatePodSpec, newArangoMemberUpdatePodSpecAction) +} + +// newArangoMemberUpdatePodSpecAction creates a new Action that implements the given +// planned ArangoMemberUpdatePodSpec action. +func newArangoMemberUpdatePodSpecAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &actionArangoMemberUpdatePodSpec{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout) + + return a +} + +// actionArangoMemberUpdatePodSpec implements an ArangoMemberUpdatePodSpec. +type actionArangoMemberUpdatePodSpec struct { + // actionImpl implement timeout and member id functions + actionImpl + + // actionEmptyCheckProgress implement check progress with empty implementation + actionEmptyCheckProgress +} + +// Start performs the start of the action. +// Returns true if the action is completely finished, false in case +// the start time needs to be recorded and a ready condition needs to be checked. +func (a *actionArangoMemberUpdatePodSpec) Start(ctx context.Context) (bool, error) { + spec := a.actionCtx.GetSpec() + status := a.actionCtx.GetStatus() + + m, found := a.actionCtx.GetMemberStatusByID(a.action.MemberID) + if !found { + log.Error().Msg("No such member") + return true, nil + } + + cache := a.actionCtx.GetCachedStatus() + + member, ok := cache.ArangoMember(m.ArangoMemberName(a.actionCtx.GetName(), a.action.Group)) + if !ok { + err := errors.Newf("ArangoMember not found") + log.Error().Err(err).Msg("ArangoMember not found") + return false, err + } + + endpoint, err := pod.GenerateMemberEndpoint(a.actionCtx.GetCachedStatus(), a.actionCtx.GetAPIObject(), spec, a.action.Group, m) + if err != nil { + log.Error().Err(err).Msg("Unable to render endpoint") + return false, err + } + + if m.Endpoint == nil || *m.Endpoint != endpoint { + // Update endpoint + m.Endpoint = &endpoint + if err := status.Members.Update(m, a.action.Group); err != nil { + log.Error().Err(err).Msg("Unable to update endpoint") + return false, err + } + } + + groupSpec := spec.GetServerGroupSpec(a.action.Group) + + imageInfo, imageFound := a.actionCtx.SelectImage(spec, status) + if !imageFound { + // Image is not found, so rotation is not needed + return true, nil + } + + if m.Image != nil { + imageInfo = *m.Image + } + + renderedPod, err := a.actionCtx.RenderPodTemplateForMember(ctx, a.actionCtx.GetCachedStatus(), spec, status, a.action.MemberID, imageInfo) + if err != nil { + log.Err(err).Msg("Error while rendering pod") + return false, err + } + + checksum, err := resources.ChecksumArangoPod(groupSpec, resources.CreatePodFromTemplate(renderedPod)) + if err != nil { + log.Err(err).Msg("Error while getting pod checksum") + return false, err + } + + template, err := api.GetArangoMemberPodTemplate(renderedPod, checksum) + if err != nil { + log.Err(err).Msg("Error while getting pod template") + return false, err + } + + if err := a.actionCtx.WithArangoMemberUpdate(context.Background(), member.GetNamespace(), member.GetName(), func(member *api.ArangoMember) bool { + if !member.Spec.Template.Equals(template) { + member.Spec.Template = template.DeepCopy() + return true + } + + return false + }); err != nil { + log.Err(err).Msg("Error while updating member") + return false, err + } + + if err := a.actionCtx.WithArangoMemberStatusUpdate(context.Background(), member.GetNamespace(), member.GetName(), func(member *api.ArangoMember, status *api.ArangoMemberStatus) bool { + if (status.Template == nil || status.Template.PodSpec == nil) && (m.PodSpecVersion == "" || m.PodSpecVersion == template.PodSpecChecksum) { + status.Template = template.DeepCopy() + } + + return true + }); err != nil { + log.Err(err).Msg("Error while updating member status") + return false, err + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/action_arango_membed_update_pod_status.go b/pkg/deployment/reconcile/action_arango_membed_update_pod_status.go new file mode 100644 index 000000000..f0688c7a8 --- /dev/null +++ b/pkg/deployment/reconcile/action_arango_membed_update_pod_status.go @@ -0,0 +1,91 @@ +// +// DISCLAIMER +// +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + "github.com/arangodb/kube-arangodb/pkg/util/errors" + "github.com/rs/zerolog/log" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/rs/zerolog" +) + +func init() { + registerAction(api.ActionTypeArangoMemberUpdatePodStatus, newArangoMemberUpdatePodStatusAction) +} + +// newArangoMemberUpdatePodStatusAction creates a new Action that implements the given +// planned ArangoMemberUpdatePodStatus action. +func newArangoMemberUpdatePodStatusAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &actionArangoMemberUpdatePodStatus{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout) + + return a +} + +// actionArangoMemberUpdatePodStatus implements an ArangoMemberUpdatePodStatus. +type actionArangoMemberUpdatePodStatus struct { + // actionImpl implement timeout and member id functions + actionImpl + + // actionEmptyCheckProgress implement check progress with empty implementation + actionEmptyCheckProgress +} + +// Start performs the start of the action. +// Returns true if the action is completely finished, false in case +// the start time needs to be recorded and a ready condition needs to be checked. +func (a *actionArangoMemberUpdatePodStatus) Start(ctx context.Context) (bool, error) { + m, found := a.actionCtx.GetMemberStatusByID(a.action.MemberID) + if !found { + log.Error().Msg("No such member") + return true, nil + } + + cache := a.actionCtx.GetCachedStatus() + + member, ok := cache.ArangoMember(m.ArangoMemberName(a.actionCtx.GetName(), a.action.Group)) + if !ok { + err := errors.Newf("ArangoMember not found") + log.Error().Err(err).Msg("ArangoMember not found") + return false, err + } + + if member.Status.Template == nil || !member.Status.Template.Equals(member.Spec.Template) { + if err := a.actionCtx.WithArangoMemberStatusUpdate(context.Background(), member.GetNamespace(), member.GetName(), func(obj *api.ArangoMember, status *api.ArangoMemberStatus) bool { + if status.Template == nil || !status.Template.Equals(member.Spec.Template) { + status.Template = member.Spec.Template.DeepCopy() + return true + } + return false + }); err != nil { + log.Err(err).Msg("Error while updating member") + return false, err + } + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/action_context.go b/pkg/deployment/reconcile/action_context.go index 3df33fcd7..3621830d3 100644 --- a/pkg/deployment/reconcile/action_context.go +++ b/pkg/deployment/reconcile/action_context.go @@ -36,7 +36,7 @@ import ( "github.com/arangodb/go-driver/agency" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" - v1 "k8s.io/api/core/v1" + core "k8s.io/api/core/v1" "github.com/arangodb/arangosync-client/client" driver "github.com/arangodb/go-driver" @@ -51,6 +51,8 @@ import ( type ActionContext interface { resources.DeploymentStatusUpdate resources.DeploymentAgencyMaintenance + resources.ArangoMemberContext + resources.DeploymentPodRenderer // GetAPIObject returns the deployment as k8s object. GetAPIObject() k8sutil.APIObject @@ -88,7 +90,7 @@ type ActionContext interface { // RemoveMemberByID removes a member with given id. RemoveMemberByID(ctx context.Context, id string) error // GetPod returns pod. - GetPod(ctx context.Context, podName string) (*v1.Pod, error) + GetPod(ctx context.Context, podName string) (*core.Pod, error) // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. DeletePod(ctx context.Context, podName string) error @@ -97,10 +99,10 @@ type ActionContext interface { DeletePvc(ctx context.Context, pvcName string) error // GetPvc returns PVC info about PVC with given name in the namespace // of the deployment. - GetPvc(ctx context.Context, pvcName string) (*v1.PersistentVolumeClaim, error) + GetPvc(ctx context.Context, pvcName string) (*core.PersistentVolumeClaim, error) // UpdatePvc update PVC with given name in the namespace // of the deployment. - UpdatePvc(ctx context.Context, pvc *v1.PersistentVolumeClaim) error + UpdatePvc(ctx context.Context, pvc *core.PersistentVolumeClaim) error // RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. RemovePodFinalizers(ctx context.Context, podName string) error @@ -139,8 +141,10 @@ type ActionContext interface { GetBackup(ctx context.Context, backup string) (*backupApi.ArangoBackup, error) // GetName receives information about a deployment name GetName() string - // GetNameget current cached state of deployment + // GetCachedStatus current cached state of deployment GetCachedStatus() inspectorInterface.Inspector + // SelectImage select currently used image by pod + SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) } // newActionContext creates a new ActionContext implementation. @@ -154,11 +158,19 @@ func newActionContext(log zerolog.Logger, context Context, cachedStatus inspecto // actionContext implements ActionContext type actionContext struct { - log zerolog.Logger context Context + log zerolog.Logger cachedStatus inspectorInterface.Inspector } +func (ac *actionContext) RenderPodForMemberFromCurrent(ctx context.Context, cachedStatus inspectorInterface.Inspector, memberID string) (*core.Pod, error) { + return ac.context.RenderPodForMemberFromCurrent(ctx, cachedStatus, memberID) +} + +func (ac *actionContext) RenderPodTemplateForMemberFromCurrent(ctx context.Context, cachedStatus inspectorInterface.Inspector, memberID string) (*core.PodTemplateSpec, error) { + return ac.context.RenderPodTemplateForMemberFromCurrent(ctx, cachedStatus, memberID) +} + func (ac *actionContext) GetAgencyMaintenanceMode(ctx context.Context) (bool, error) { return ac.context.GetAgencyMaintenanceMode(ctx) } @@ -167,6 +179,26 @@ func (ac *actionContext) SetAgencyMaintenanceMode(ctx context.Context, enabled b return ac.context.SetAgencyMaintenanceMode(ctx, enabled) } +func (ac *actionContext) WithArangoMemberUpdate(ctx context.Context, namespace, name string, action resources.ArangoMemberUpdateFunc) error { + return ac.context.WithArangoMemberUpdate(ctx, namespace, name, action) +} + +func (ac *actionContext) WithArangoMemberStatusUpdate(ctx context.Context, namespace, name string, action resources.ArangoMemberStatusUpdateFunc) error { + return ac.context.WithArangoMemberStatusUpdate(ctx, namespace, name, action) +} + +func (ac *actionContext) RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) { + return ac.context.RenderPodForMember(ctx, cachedStatus, spec, status, memberID, imageInfo) +} + +func (ac *actionContext) RenderPodTemplateForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.PodTemplateSpec, error) { + return ac.context.RenderPodTemplateForMember(ctx, cachedStatus, spec, status, memberID, imageInfo) +} + +func (ac *actionContext) SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) { + return ac.context.SelectImage(spec, status) +} + func (ac *actionContext) GetCachedStatus() inspectorInterface.Inspector { return ac.cachedStatus } @@ -209,7 +241,7 @@ func (ac *actionContext) GetAPIObject() k8sutil.APIObject { return ac.context.GetAPIObject() } -func (ac *actionContext) UpdatePvc(ctx context.Context, pvc *v1.PersistentVolumeClaim) error { +func (ac *actionContext) UpdatePvc(ctx context.Context, pvc *core.PersistentVolumeClaim) error { return ac.context.UpdatePvc(ctx, pvc) } @@ -217,7 +249,7 @@ func (ac *actionContext) CreateEvent(evt *k8sutil.Event) { ac.context.CreateEvent(evt) } -func (ac *actionContext) GetPvc(ctx context.Context, pvcName string) (*v1.PersistentVolumeClaim, error) { +func (ac *actionContext) GetPvc(ctx context.Context, pvcName string) (*core.PersistentVolumeClaim, error) { return ac.context.GetPvc(ctx, pvcName) } @@ -346,7 +378,7 @@ func (ac *actionContext) RemoveMemberByID(ctx context.Context, id string) error } // GetPod returns pod. -func (ac *actionContext) GetPod(ctx context.Context, podName string) (*v1.Pod, error) { +func (ac *actionContext) GetPod(ctx context.Context, podName string) (*core.Pod, error) { if pod, err := ac.context.GetPod(ctx, podName); err != nil { return nil, errors.WithStack(err) } else { diff --git a/pkg/deployment/reconcile/context.go b/pkg/deployment/reconcile/context.go index d773316f2..1bc2e3c21 100644 --- a/pkg/deployment/reconcile/context.go +++ b/pkg/deployment/reconcile/context.go @@ -37,13 +37,15 @@ import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/util/arangod/conn" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" - inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" ) // Context provides methods to the reconcile package. type Context interface { resources.DeploymentStatusUpdate resources.DeploymentAgencyMaintenance + resources.ArangoMemberContext + resources.DeploymentPodRenderer + resources.DeploymentImageManager // GetAPIObject returns the deployment as k8s object. GetAPIObject() k8sutil.APIObject @@ -112,10 +114,6 @@ type Context interface { EnableScalingCluster(ctx context.Context) error // GetAgencyData object for key path GetAgencyData(ctx context.Context, i interface{}, keyParts ...string) error - // Renders Pod definition for member - RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*v1.Pod, error) - // SelectImage select currently used image by pod - SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) // SecretsInterface return secret interface SecretsInterface() k8sutil.SecretInterface // GetBackup receives information about a backup resource diff --git a/pkg/deployment/reconcile/plan_builder_appender.go b/pkg/deployment/reconcile/plan_builder_appender.go new file mode 100644 index 000000000..5a68a3f73 --- /dev/null +++ b/pkg/deployment/reconcile/plan_builder_appender.go @@ -0,0 +1,95 @@ +// +// DISCLAIMER +// +// Copyright 2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" +) + +func newPlanAppender(pb WithPlanBuilder, current api.Plan) PlanAppender { + return planAppenderType{ + current: current, + pb: pb, + } +} + +type PlanAppender interface { + Apply(pb planBuilder) PlanAppender + ApplyWithCondition(c planBuilderCondition, pb planBuilder) PlanAppender + ApplySubPlan(pb planBuilderSubPlan, plans ...planBuilder) PlanAppender + + ApplyIfEmpty(pb planBuilder) PlanAppender + ApplyWithConditionIfEmpty(c planBuilderCondition, pb planBuilder) PlanAppender + ApplySubPlanIfEmpty(pb planBuilderSubPlan, plans ...planBuilder) PlanAppender + + Plan() api.Plan +} + +type planAppenderType struct { + pb WithPlanBuilder + current api.Plan +} + +func (p planAppenderType) Plan() api.Plan { + return p.current +} + +func (p planAppenderType) ApplyIfEmpty(pb planBuilder) PlanAppender { + if p.current.IsEmpty() { + return p.Apply(pb) + } + return p +} + +func (p planAppenderType) ApplyWithConditionIfEmpty(c planBuilderCondition, pb planBuilder) PlanAppender { + if p.current.IsEmpty() { + return p.ApplyWithCondition(c, pb) + } + return p +} + +func (p planAppenderType) ApplySubPlanIfEmpty(pb planBuilderSubPlan, plans ...planBuilder) PlanAppender { + if p.current.IsEmpty() { + return p.ApplySubPlan(pb, plans...) + } + return p +} + +func (p planAppenderType) new(plan api.Plan) planAppenderType { + return planAppenderType{ + pb: p.pb, + current: append(p.current, plan...), + } +} + +func (p planAppenderType) Apply(pb planBuilder) PlanAppender { + return p.new(p.pb.Apply(pb)) +} + +func (p planAppenderType) ApplyWithCondition(c planBuilderCondition, pb planBuilder) PlanAppender { + return p.new(p.pb.ApplyWithCondition(c, pb)) +} + +func (p planAppenderType) ApplySubPlan(pb planBuilderSubPlan, plans ...planBuilder) PlanAppender { + return p.new(p.pb.ApplySubPlan(pb, plans...)) +} diff --git a/pkg/deployment/reconcile/plan_builder_context.go b/pkg/deployment/reconcile/plan_builder_context.go index 30efe8106..715792b2a 100644 --- a/pkg/deployment/reconcile/plan_builder_context.go +++ b/pkg/deployment/reconcile/plan_builder_context.go @@ -27,10 +27,8 @@ import ( "github.com/arangodb/kube-arangodb/pkg/deployment/resources" - "github.com/arangodb/kube-arangodb/pkg/util/arangod/conn" - inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" - "github.com/arangodb/go-driver/agency" + "github.com/arangodb/kube-arangodb/pkg/util/arangod/conn" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1" @@ -45,6 +43,9 @@ import ( type PlanBuilderContext interface { resources.DeploymentStatusUpdate resources.DeploymentAgencyMaintenance + resources.ArangoMemberContext + resources.DeploymentPodRenderer + resources.DeploymentImageManager // GetTLSKeyfile returns the keyfile encoded TLS certificate+key for // the given member. @@ -64,10 +65,6 @@ type PlanBuilderContext interface { GetSpec() api.DeploymentSpec // GetAgencyData object for key path GetAgencyData(ctx context.Context, i interface{}, keyParts ...string) error - // Renders Pod definition for member - RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) - // SelectImage select currently used image by pod - SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) // GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server), // creating one if needed. GetDatabaseClient(ctx context.Context) (driver.Client, error) diff --git a/pkg/deployment/reconcile/plan_builder_factory.go b/pkg/deployment/reconcile/plan_builder_factory.go new file mode 100644 index 000000000..41a348d0c --- /dev/null +++ b/pkg/deployment/reconcile/plan_builder_factory.go @@ -0,0 +1,94 @@ +// +// DISCLAIMER +// +// Copyright 2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" + "github.com/rs/zerolog" +) + +type planBuilder func(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan + +type planBuilderCondition func(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) bool + +type planBuilderSubPlan func(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspectorInterface.Inspector, context PlanBuilderContext, w WithPlanBuilder, plans ...planBuilder) api.Plan + +func NewWithPlanBuilder(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) WithPlanBuilder { + return &withPlanBuilder{ + ctx: ctx, + log: log, + apiObject: apiObject, + spec: spec, + status: status, + cachedStatus: cachedStatus, + context: context, + } +} + +type WithPlanBuilder interface { + Apply(p planBuilder) api.Plan + ApplyWithCondition(c planBuilderCondition, p planBuilder) api.Plan + ApplySubPlan(p planBuilderSubPlan, plans ...planBuilder) api.Plan +} + +type withPlanBuilder struct { + ctx context.Context + log zerolog.Logger + apiObject k8sutil.APIObject + spec api.DeploymentSpec + status api.DeploymentStatus + cachedStatus inspectorInterface.Inspector + context PlanBuilderContext +} + +func (w withPlanBuilder) ApplyWithCondition(c planBuilderCondition, p planBuilder) api.Plan { + if !c(w.ctx, w.log, w.apiObject, w.spec, w.status, w.cachedStatus, w.context) { + return api.Plan{} + } + + return w.Apply(p) +} + +func (w withPlanBuilder) ApplySubPlan(p planBuilderSubPlan, plans ...planBuilder) api.Plan { + return p(w.ctx, w.log, w.apiObject, w.spec, w.status, w.cachedStatus, w.context, w, plans...) +} + +func (w withPlanBuilder) Apply(p planBuilder) api.Plan { + return p(w.ctx, w.log, w.apiObject, w.spec, w.status, w.cachedStatus, w.context) +} diff --git a/pkg/deployment/reconcile/plan_builder_high.go b/pkg/deployment/reconcile/plan_builder_high.go index b32919902..9ea47fc4e 100644 --- a/pkg/deployment/reconcile/plan_builder_high.go +++ b/pkg/deployment/reconcile/plan_builder_high.go @@ -25,11 +25,13 @@ package reconcile import ( "context" + "github.com/arangodb/kube-arangodb/pkg/apis/deployment" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" "github.com/rs/zerolog" + core "k8s.io/api/core/v1" ) func (d *Reconciler) CreateHighPlan(ctx context.Context, cachedStatus inspectorInterface.Inspector) (error, bool) { @@ -77,21 +79,33 @@ func createHighPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.A return currentPlan, false } - // Check for various scenario's + return newPlanAppender(NewWithPlanBuilder(ctx, log, apiObject, spec, status, cachedStatus, builderCtx), nil). + ApplyIfEmpty(updateMemberPodTemplateSpec). + ApplyIfEmpty(updateMemberPhasePlan). + ApplyIfEmpty(createCleanOutPlan). + ApplyIfEmpty(updateMemberRotationFlag). + Plan(), true +} + +// updateMemberPodTemplateSpec creates plan to update member Spec +func updateMemberPodTemplateSpec(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { var plan api.Plan - pb := NewWithPlanBuilder(ctx, log, apiObject, spec, status, cachedStatus, builderCtx) + // Update member specs + status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { + for _, m := range members { + if reason, changed := arangoMemberPodTemplateNeedsUpdate(ctx, log, apiObject, spec, group, status, m, cachedStatus, context); changed { + plan = append(plan, api.NewAction(api.ActionTypeArangoMemberUpdatePodSpec, group, m.ID, reason)) + } + } - if plan.IsEmpty() { - plan = pb.Apply(updateMemberPhasePlan) - } + return nil + }) - if plan.IsEmpty() { - plan = pb.Apply(createCleanOutPlan) - } - - // Return plan - return plan, true + return plan } // updateMemberPhasePlan creates plan to update member phase @@ -106,6 +120,7 @@ func updateMemberPhasePlan(ctx context.Context, if m.Phase == api.MemberPhaseNone { plan = append(plan, api.NewAction(api.ActionTypeMemberRIDUpdate, group, m.ID, "Regenerate member RID"), + api.NewAction(api.ActionTypeArangoMemberUpdatePodStatus, group, m.ID, "Propagating status of pod"), api.NewAction(api.ActionTypeMemberPhaseUpdate, group, m.ID, "Move to Pending phase").AddParam(ActionTypeMemberPhaseUpdatePhaseKey, api.MemberPhasePending.String())) } @@ -116,3 +131,67 @@ func updateMemberPhasePlan(ctx context.Context, return plan } + +func updateMemberRotationFlag(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { + var plan api.Plan + + status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { + for _, m := range list { + p, found := cachedStatus.Pod(m.PodName) + if !found { + continue + } + + if required, reason := updateMemberRotationFlagConditionCheck(log, apiObject, spec, cachedStatus, m, group, p); required { + log.Info().Msgf(reason) + plan = append(plan, restartMemberConditionPlan(group, m.ID, reason)...) + } + } + + return nil + }) + + return plan +} + +func restartMemberConditionPlan(group api.ServerGroup, memberID string, reason string) api.Plan { + return api.Plan{ + api.NewAction(api.ActionTypeSetMemberCondition, group, memberID, reason).AddParam(api.ConditionTypePendingRestart.String(), "T"), + } +} + +func tlsRotateConditionPlan(group api.ServerGroup, memberID string, reason string) api.Plan { + return api.Plan{ + api.NewAction(api.ActionTypeSetMemberCondition, group, memberID, reason).AddParam(api.ConditionTypePendingTLSRotation.String(), "T"), + } +} + +func updateMemberRotationFlagConditionCheck(log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, cachedStatus inspectorInterface.Inspector, m api.MemberStatus, group api.ServerGroup, p *core.Pod) (bool, string) { + if m.Conditions.IsTrue(api.ConditionTypePendingRestart) { + return false, "" + } + + if m.Conditions.IsTrue(api.ConditionTypePendingTLSRotation) { + return true, "TLS Rotation pending" + } + + pvc, exists := cachedStatus.PersistentVolumeClaim(m.PersistentVolumeClaimName) + if exists { + if k8sutil.IsPersistentVolumeClaimFileSystemResizePending(pvc) { + return true, "PVC Resize pending" + } + } + + if changed, reason := podNeedsRotation(log, apiObject, p, spec, group, m, cachedStatus); changed { + return true, reason + } + + if _, ok := p.Annotations[deployment.ArangoDeploymentPodRotateAnnotation]; ok { + return true, "Rotation flag present" + } + + return false, "" +} diff --git a/pkg/deployment/reconcile/plan_builder_normal.go b/pkg/deployment/reconcile/plan_builder_normal.go index de27a9753..8d700982e 100644 --- a/pkg/deployment/reconcile/plan_builder_normal.go +++ b/pkg/deployment/reconcile/plan_builder_normal.go @@ -77,13 +77,49 @@ func createNormalPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil return currentPlan, false } - // Fetch agency plan - agencyPlan, agencyErr := fetchAgency(ctx, spec, status, builderCtx) + return newPlanAppender(NewWithPlanBuilder(ctx, log, apiObject, spec, status, cachedStatus, builderCtx), nil). + // Check for failed members + ApplyIfEmpty(createMemberFailedRestorePlan). + // Check for cleaned out dbserver in created state + ApplyIfEmpty(createRemoveCleanedDBServersPlan). + // Update status + ApplySubPlanIfEmpty(createEncryptionKeyStatusPropagatedFieldUpdate, createEncryptionKeyStatusUpdate). + ApplyIfEmpty(createTLSStatusUpdate). + ApplyIfEmpty(createJWTStatusUpdate). + // Check for scale up/down + ApplyIfEmpty(createScaleMemberPlan). + // Check for members to be removed + ApplyIfEmpty(createReplaceMemberPlan). + // Check for the need to rotate one or more members + ApplyIfEmpty(createRotateOrUpgradePlan). + // Disable maintenance if upgrade process was done. Upgrade task throw IDLE Action if upgrade is pending + ApplyIfEmpty(createMaintenanceManagementPlan). + // Add keys + ApplySubPlanIfEmpty(createEncryptionKeyStatusPropagatedFieldUpdate, createEncryptionKey). + ApplyIfEmpty(createJWTKeyUpdate). + ApplySubPlanIfEmpty(createTLSStatusPropagatedFieldUpdate, createCARenewalPlan). + ApplySubPlanIfEmpty(createTLSStatusPropagatedFieldUpdate, createCAAppendPlan). + ApplyIfEmpty(createKeyfileRenewalPlan). + ApplyIfEmpty(createRotateServerStoragePlan). + ApplySubPlanIfEmpty(createTLSStatusPropagatedFieldUpdate, createRotateTLSServerSNIPlan). + ApplyIfEmpty(createRestorePlan). + ApplySubPlanIfEmpty(createEncryptionKeyStatusPropagatedFieldUpdate, createEncryptionKeyCleanPlan). + ApplySubPlanIfEmpty(createTLSStatusPropagatedFieldUpdate, createCACleanPlan). + ApplyIfEmpty(createClusterOperationPlan). + // Final + ApplyIfEmpty(createTLSStatusPropagated). + ApplyIfEmpty(createBootstrapPlan). + Plan(), true +} - // Check for various scenario's +func createMemberFailedRestorePlan(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { var plan api.Plan - pb := NewWithPlanBuilder(ctx, log, apiObject, spec, status, cachedStatus, builderCtx) + // Fetch agency plan + agencyPlan, agencyErr := fetchAgency(ctx, spec, status, context) // Check for members in failed state status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { @@ -149,113 +185,29 @@ func createNormalPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil // Ensure that we were able to get agency info if len(plan) == 0 && agencyErr != nil { log.Err(agencyErr).Msg("unable to build further plan without access to agency") - return append(plan, - api.NewAction(api.ActionTypeIdle, api.ServerGroupUnknown, "")), true + plan = append(plan, + api.NewAction(api.ActionTypeIdle, api.ServerGroupUnknown, "")) } - // Check for cleaned out dbserver in created state + return plan +} + +func createRemoveCleanedDBServersPlan(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { for _, m := range status.Members.DBServers { - if plan.IsEmpty() && m.Phase.IsCreatedOrDrain() && m.Conditions.IsTrue(api.ConditionTypeCleanedOut) { + if m.Phase.IsCreatedOrDrain() && m.Conditions.IsTrue(api.ConditionTypeCleanedOut) { log.Debug(). Str("id", m.ID). Str("role", api.ServerGroupDBServers.AsRole()). Msg("Creating dbserver replacement plan because server is cleanout in created phase") - plan = append(plan, + return api.Plan{ api.NewAction(api.ActionTypeRemoveMember, api.ServerGroupDBServers, m.ID), api.NewAction(api.ActionTypeAddMember, api.ServerGroupDBServers, ""), - ) + } } } - // Update status - if plan.IsEmpty() { - plan = pb.ApplySubPlan(createEncryptionKeyStatusPropagatedFieldUpdate, createEncryptionKeyStatusUpdate) - } - - if plan.IsEmpty() { - plan = pb.Apply(createTLSStatusUpdate) - } - - if plan.IsEmpty() { - plan = pb.Apply(createJWTStatusUpdate) - } - - // Check for scale up/down - if plan.IsEmpty() { - plan = pb.Apply(createScaleMemberPlan) - } - - // Check for members to be removed - if plan.IsEmpty() { - plan = pb.Apply(createReplaceMemberPlan) - } - - // Check for the need to rotate one or more members - if plan.IsEmpty() { - plan = pb.Apply(createRotateOrUpgradePlan) - } - - // Disable maintenance if upgrade process was done. Upgrade task throw IDLE Action if upgrade is pending - if plan.IsEmpty() { - plan = pb.Apply(createMaintenanceManagementPlan) - } - - // Add keys - if plan.IsEmpty() { - plan = pb.ApplySubPlan(createEncryptionKeyStatusPropagatedFieldUpdate, createEncryptionKey) - } - - if plan.IsEmpty() { - plan = pb.Apply(createJWTKeyUpdate) - } - - if plan.IsEmpty() { - plan = pb.ApplySubPlan(createTLSStatusPropagatedFieldUpdate, createCARenewalPlan) - } - - if plan.IsEmpty() { - plan = pb.ApplySubPlan(createTLSStatusPropagatedFieldUpdate, createCAAppendPlan) - } - - if plan.IsEmpty() { - plan = pb.Apply(createKeyfileRenewalPlan) - } - - // Check for changes storage classes or requirements - if plan.IsEmpty() { - plan = pb.Apply(createRotateServerStoragePlan) - } - - if plan.IsEmpty() { - plan = pb.ApplySubPlan(createTLSStatusPropagatedFieldUpdate, createRotateTLSServerSNIPlan) - } - - if plan.IsEmpty() { - plan = pb.Apply(createRestorePlan) - } - - if plan.IsEmpty() { - plan = pb.ApplySubPlan(createEncryptionKeyStatusPropagatedFieldUpdate, createEncryptionKeyCleanPlan) - } - - if plan.IsEmpty() { - plan = pb.ApplySubPlan(createTLSStatusPropagatedFieldUpdate, createCACleanPlan) - } - - if plan.IsEmpty() { - plan = pb.Apply(createClusterOperationPlan) - } - - // Final - - if plan.IsEmpty() { - plan = pb.Apply(createTLSStatusPropagated) - } - - if plan.IsEmpty() { - plan = pb.Apply(createBootstrapPlan) - } - - // Return plan - return plan, true + return nil } diff --git a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go index ee3bc17b1..f0616b11e 100644 --- a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go +++ b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go @@ -27,8 +27,6 @@ import ( "github.com/arangodb/kube-arangodb/pkg/deployment/features" - json "github.com/json-iterator/go" - "github.com/arangodb/kube-arangodb/pkg/deployment/pod" "github.com/arangodb/kube-arangodb/pkg/deployment/resources" @@ -76,7 +74,7 @@ func createRotateOrUpgradePlan(ctx context.Context, cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { var plan api.Plan - newPlan, idle := createRotateOrUpgradePlanInternal(ctx, log, apiObject, spec, status, cachedStatus, context) + newPlan, idle := createRotateOrUpgradePlanInternal(log, apiObject, spec, status, cachedStatus, context) if idle { plan = append(plan, api.NewAction(api.ActionTypeIdle, api.ServerGroupUnknown, "")) @@ -86,8 +84,7 @@ func createRotateOrUpgradePlan(ctx context.Context, return plan } -func createRotateOrUpgradePlanInternal(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, - status api.DeploymentStatus, cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) (api.Plan, bool) { +func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) (api.Plan, bool) { var newPlan api.Plan var upgradeNotAllowed bool @@ -95,18 +92,12 @@ func createRotateOrUpgradePlanInternal(ctx context.Context, log zerolog.Logger, var fromLicense, toLicense upgraderules.License status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { - for _, m := range members { if m.Phase != api.MemberPhaseCreated || m.PodName == "" { // Only rotate when phase is created continue } - pod, found := cachedStatus.Pod(m.PodName) - if !found { - continue - } - // Got pod, compare it with what it should be decision := podNeedsUpgrading(log, m, spec, status.Images) if decision.Hold { @@ -133,10 +124,8 @@ func createRotateOrUpgradePlanInternal(ctx context.Context, log zerolog.Logger, newPlan = createUpgradeMemberPlan(log, m, group, "Version upgrade", spec, status, !decision.AutoUpgradeNeeded) } else { - // Use new level of rotate logic - rotNeeded, reason := podNeedsRotation(ctx, log, apiObject, pod, spec, group, status, m, cachedStatus, context) - if rotNeeded { - newPlan = createRotateMemberPlan(log, m, group, reason) + if m.Conditions.IsTrue(api.ConditionTypePendingRestart) { + newPlan = createRotateMemberPlan(log, m, group, "Restart flag present") } } @@ -170,11 +159,6 @@ func createRotateOrUpgradePlanInternal(ctx context.Context, log zerolog.Logger, newPlan = api.Plan{api.NewAction(api.ActionTypeMarkToRemoveMember, group, m.ID, "Replace flag present")} continue } - - if _, ok := pod.Annotations[deployment.ArangoDeploymentPodRotateAnnotation]; ok { - newPlan = createRotateMemberPlan(log, m, group, "Rotation flag present") - continue - } } } @@ -298,14 +282,57 @@ func memberImageInfo(spec api.DeploymentSpec, status api.MemberStatus, images ap return api.ImageInfo{}, false } +func getPodDetails(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, + group api.ServerGroup, status api.DeploymentStatus, m api.MemberStatus, + cachedStatus inspectorInterface.Inspector, planCtx PlanBuilderContext) (string, *core.Pod, *api.ArangoMember, bool) { + imageInfo, imageFound := planCtx.SelectImageForMember(spec, status, m) + if !imageFound { + // Image is not found, so rotation is not needed + return "", nil, nil, false + } + + member, ok := cachedStatus.ArangoMember(m.ArangoMemberName(apiObject.GetName(), group)) + if !ok { + return "", nil, nil, false + } + + groupSpec := spec.GetServerGroupSpec(group) + + renderedPod, err := planCtx.RenderPodForMember(ctx, cachedStatus, spec, status, m.ID, imageInfo) + if err != nil { + log.Err(err).Msg("Error while rendering pod") + return "", nil, nil, false + } + + checksum, err := resources.ChecksumArangoPod(groupSpec, renderedPod) + if err != nil { + log.Err(err).Msg("Error while getting pod checksum") + return "", nil, nil, false + } + + return checksum, renderedPod, member, true +} + +// arangoMemberPodTemplateNeedsUpdate returns true when the specification of the +// given pod differs from what it should be according to the +// given deployment spec. +// When true is returned, a reason for the rotation is already returned. +func arangoMemberPodTemplateNeedsUpdate(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, + group api.ServerGroup, status api.DeploymentStatus, m api.MemberStatus, + cachedStatus inspectorInterface.Inspector, planCtx PlanBuilderContext) (string, bool) { + checksum, _, member, valid := getPodDetails(ctx, log, apiObject, spec, group, status, m, cachedStatus, planCtx) + if valid && !member.Spec.Template.EqualPodSpecChecksum(checksum) { + return "Pod Spec changed", true + } + + return "", false +} + // podNeedsRotation returns true when the specification of the // given pod differs from what it should be according to the // given deployment spec. // When true is returned, a reason for the rotation is already returned. -func podNeedsRotation(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, p *core.Pod, spec api.DeploymentSpec, - group api.ServerGroup, status api.DeploymentStatus, m api.MemberStatus, - cachedStatus inspectorInterface.Inspector, planCtx PlanBuilderContext) (bool, string) { - +func podNeedsRotation(log zerolog.Logger, apiObject k8sutil.APIObject, p *core.Pod, spec api.DeploymentSpec, group api.ServerGroup, m api.MemberStatus, cachedStatus inspectorInterface.Inspector) (bool, string) { if m.PodUID != p.UID { return true, "Pod UID does not match, this pod is not managed by Operator. Recreating" } @@ -314,34 +341,17 @@ func podNeedsRotation(ctx context.Context, log zerolog.Logger, apiObject k8sutil return true, "Pod Spec Version is nil - recreating pod" } - imageInfo, imageFound := planCtx.SelectImage(spec, status) - if !imageFound { - // Image is not found, so rotation is not needed + member, ok := cachedStatus.ArangoMember(m.ArangoMemberName(apiObject.GetName(), group)) + if !ok { return false, "" } - if m.Image != nil { - imageInfo = *m.Image - } - - groupSpec := spec.GetServerGroupSpec(group) - - renderedPod, err := planCtx.RenderPodForMember(ctx, cachedStatus, spec, status, m.ID, imageInfo) - if err != nil { - log.Err(err).Msg("Error while rendering pod") + if member.Status.Template == nil { return false, "" } - checksum, err := resources.ChecksumArangoPod(groupSpec, renderedPod) - if err != nil { - log.Err(err).Msg("Error while getting pod checksum") - return false, "" - } - - if m.PodSpecVersion != checksum { - if _, err := json.Marshal(renderedPod); err == nil { - log.Info().Str("id", m.ID).Str("Before", m.PodSpecVersion).Str("After", checksum).Msgf("XXXXXXXXXXX Pod needs rotation - checksum does not match") - } + if member.Status.Template.RotationNeeded(member.Spec.Template) { + log.Info().Str("id", m.ID).Str("Before", m.PodSpecVersion).Msgf("Pod needs rotation - templates does not match") return true, "Pod needs rotation - checksum does not match" } diff --git a/pkg/deployment/reconcile/plan_builder_storage.go b/pkg/deployment/reconcile/plan_builder_storage.go index 16af5e3dd..ca7659f3a 100644 --- a/pkg/deployment/reconcile/plan_builder_storage.go +++ b/pkg/deployment/reconcile/plan_builder_storage.go @@ -61,6 +61,7 @@ func createRotateServerStoragePlan(ctx context.Context, } groupSpec := spec.GetServerGroupSpec(group) storageClassName := groupSpec.GetStorageClassName() + // Load PVC pvc, exists := cachedStatus.PersistentVolumeClaim(m.PersistentVolumeClaimName) if !exists { @@ -98,9 +99,6 @@ func createRotateServerStoragePlan(ctx context.Context, // Only agents & dbservers are allowed to change their storage class. context.CreateEvent(k8sutil.NewCannotChangeStorageClassEvent(apiObject, m.ID, group.AsRole(), "Not supported")) } - } else if k8sutil.IsPersistentVolumeClaimFileSystemResizePending(pvc) { - // rotation needed - plan = createRotateMemberPlan(log, m, group, "Filesystem resize pending") } else { if groupSpec.HasVolumeClaimTemplate() { res := groupSpec.GetVolumeClaimTemplate().Spec.Resources.Requests diff --git a/pkg/deployment/reconcile/plan_builder_test.go b/pkg/deployment/reconcile/plan_builder_test.go index c40772fda..f5bc2d98a 100644 --- a/pkg/deployment/reconcile/plan_builder_test.go +++ b/pkg/deployment/reconcile/plan_builder_test.go @@ -75,6 +75,30 @@ type testContext struct { RecordedEvent *k8sutil.Event } +func (c *testContext) RenderPodForMemberFromCurrent(ctx context.Context, cachedStatus inspectorInterface.Inspector, memberID string) (*core.Pod, error) { + panic("implement me") +} + +func (c *testContext) RenderPodTemplateForMemberFromCurrent(ctx context.Context, cachedStatus inspectorInterface.Inspector, memberID string) (*core.PodTemplateSpec, error) { + return &core.PodTemplateSpec{}, nil +} + +func (c *testContext) SelectImageForMember(spec api.DeploymentSpec, status api.DeploymentStatus, member api.MemberStatus) (api.ImageInfo, bool) { + return c.SelectImage(spec, status) +} + +func (c *testContext) RenderPodTemplateForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.PodTemplateSpec, error) { + panic("implement me") +} + +func (c *testContext) WithArangoMemberUpdate(ctx context.Context, namespace, name string, action resources.ArangoMemberUpdateFunc) error { + panic("implement me") +} + +func (c *testContext) WithArangoMemberStatusUpdate(ctx context.Context, namespace, name string, action resources.ArangoMemberStatusUpdateFunc) error { + panic("implement me") +} + func (c *testContext) GetAgencyMaintenanceMode(ctx context.Context) (bool, error) { panic("implement me") } @@ -108,7 +132,7 @@ func (c *testContext) GetAuthentication() conn.Auth { } func (c *testContext) RenderPodForMember(_ context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) { - panic("implement me") + return &core.Pod{}, nil } func (c *testContext) GetName() string { @@ -124,7 +148,12 @@ func (c *testContext) SecretsInterface() k8sutil.SecretInterface { } func (c *testContext) SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) { - panic("implement me") + return api.ImageInfo{ + Image: "", + ImageID: "", + ArangoDBVersion: "", + Enterprise: false, + }, true } func (c *testContext) UpdatePvc(_ context.Context, pvc *core.PersistentVolumeClaim) error { @@ -528,28 +557,60 @@ func (l *LastLogRecord) Run(e *zerolog.Event, level zerolog.Level, msg string) { l.msg = msg } +type testCase struct { + Name string + context *testContext + Helper func(*api.ArangoDeployment) + ExpectedError error + ExpectedPlan api.Plan + ExpectedHighPlan api.Plan + ExpectedLog string + ExpectedEvent *k8sutil.Event + + Pods map[string]*core.Pod + Secrets map[string]*core.Secret + Services map[string]*core.Service + PVCS map[string]*core.PersistentVolumeClaim + ServiceAccounts map[string]*core.ServiceAccount + PDBS map[string]*policy.PodDisruptionBudget + ServiceMonitors map[string]*monitoring.ServiceMonitor + ArangoMembers map[string]*api.ArangoMember + + Extender func(t *testing.T, r *Reconciler, c *testCase) +} + +func (t testCase) Inspector() inspectorInterface.Inspector { + return inspector.NewInspectorFromData(t.Pods, t.Secrets, t.PVCS, t.Services, t.ServiceAccounts, t.PDBS, t.ServiceMonitors, t.ArangoMembers) +} + func TestCreatePlan(t *testing.T) { // Arrange threeCoordinators := api.MemberStatusList{ { - ID: "1", + ID: "1", + PodName: "coordinator1", }, { - ID: "2", + ID: "2", + PodName: "coordinator2", }, { - ID: "3", + ID: "3", + PodName: "coordinator3", }, } threeDBServers := api.MemberStatusList{ { - ID: "1", + ID: "1", + PodName: "dbserver1", }, { - ID: "2", + ID: "2", + PodName: "dbserver2", }, { - ID: "3", + ID: "3", + PodName: "dbserver3", }, } @@ -574,24 +635,7 @@ func TestCreatePlan(t *testing.T) { addAgentsToStatus(t, &deploymentTemplate.Status, 3) deploymentTemplate.Spec.SetDefaults("createPlanTest") - testCases := []struct { - Name string - context *testContext - Helper func(*api.ArangoDeployment) - ExpectedError error - ExpectedPlan api.Plan - ExpectedLog string - ExpectedEvent *k8sutil.Event - - Pods map[string]*core.Pod - Secrets map[string]*core.Secret - Services map[string]*core.Service - PVCS map[string]*core.PersistentVolumeClaim - ServiceAccounts map[string]*core.ServiceAccount - PDBS map[string]*policy.PodDisruptionBudget - ServiceMonitors map[string]*monitoring.ServiceMonitor - ArangoMembers map[string]*api.ArangoMember - }{ + testCases := []testCase{ { Name: "Can not create plan for single deployment", context: &testContext{ @@ -735,6 +779,52 @@ func TestCreatePlan(t *testing.T) { }, }, }, + Extender: func(t *testing.T, r *Reconciler, c *testCase) { + // Add ArangoMember + builderCtx := newPlanBuilderContext(r.context) + + template, err := builderCtx.RenderPodTemplateForMemberFromCurrent(context.Background(), c.Inspector(), c.context.ArangoDeployment.Status.Members.Agents[0].ID) + require.NoError(t, err) + + checksum, err := resources.ChecksumArangoPod(c.context.ArangoDeployment.Spec.Agents, resources.CreatePodFromTemplate(template)) + require.NoError(t, err) + + templateSpec, err := api.GetArangoMemberPodTemplate(template, checksum) + require.NoError(t, err) + + name := c.context.ArangoDeployment.Status.Members.Agents[0].ArangoMemberName(c.context.ArangoDeployment.Name, api.ServerGroupAgents) + + c.ArangoMembers = map[string]*api.ArangoMember{ + name: { + ObjectMeta: meta.ObjectMeta{ + Name: name, + }, + Spec: api.ArangoMemberSpec{ + Template: templateSpec, + }, + Status: api.ArangoMemberStatus{ + Template: templateSpec, + }, + }, + } + + c.Pods = map[string]*core.Pod{ + c.context.ArangoDeployment.Status.Members.Agents[0].PodName: { + ObjectMeta: meta.ObjectMeta{ + Name: c.context.ArangoDeployment.Status.Members.Agents[0].PodName, + }, + }, + } + + require.NoError(t, c.context.ArangoDeployment.Status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { + for _, m := range list { + m.Phase = api.MemberPhaseCreated + require.NoError(t, c.context.ArangoDeployment.Status.Members.Update(m, group)) + } + + return nil + })) + }, context: &testContext{ ArangoDeployment: deploymentTemplate.DeepCopy(), }, @@ -750,14 +840,10 @@ func TestCreatePlan(t *testing.T) { ad.Status.Members.Agents[0].Phase = api.MemberPhaseCreated ad.Status.Members.Agents[0].PersistentVolumeClaimName = "pvc_test" }, - ExpectedPlan: []api.Action{ - api.NewAction(api.ActionTypeCleanTLSKeyfileCertificate, api.ServerGroupAgents, "", "Remove server keyfile and enforce renewal/recreation"), - api.NewAction(api.ActionTypeResignLeadership, api.ServerGroupAgents, ""), - api.NewAction(api.ActionTypeRotateMember, api.ServerGroupAgents, ""), - api.NewAction(api.ActionTypeWaitForMemberUp, api.ServerGroupAgents, ""), - api.NewAction(api.ActionTypeWaitForMemberInSync, api.ServerGroupAgents, ""), + ExpectedHighPlan: []api.Action{ + api.NewAction(api.ActionTypeSetMemberCondition, api.ServerGroupAgents, deploymentTemplate.Status.Members.Agents[0].ID, "PVC Resize pending"), }, - ExpectedLog: "Creating rotation plan", + ExpectedLog: "PVC Resize pending", }, { Name: "Agent in failed state", @@ -850,11 +936,16 @@ func TestCreatePlan(t *testing.T) { logger := zerolog.New(ioutil.Discard).Hook(h) r := NewReconciler(logger, testCase.context) + if testCase.Extender != nil { + testCase.Extender(t, r, &testCase) + } + // Act if testCase.Helper != nil { testCase.Helper(testCase.context.ArangoDeployment) } - err, _ := r.CreatePlan(ctx, inspector.NewInspectorFromData(testCase.Pods, testCase.Secrets, testCase.PVCS, testCase.Services, testCase.ServiceAccounts, testCase.PDBS, testCase.ServiceMonitors, testCase.ArangoMembers)) + + err, _ := r.CreatePlan(ctx, testCase.Inspector()) // Assert if testCase.ExpectedEvent != nil { @@ -873,10 +964,25 @@ func TestCreatePlan(t *testing.T) { require.NoError(t, err) status, _ := testCase.context.GetStatus() + + if len(testCase.ExpectedHighPlan) > 0 { + require.Len(t, status.HighPriorityPlan, len(testCase.ExpectedHighPlan)) + for i, v := range testCase.ExpectedHighPlan { + assert.Equal(t, v.Type, status.HighPriorityPlan[i].Type) + assert.Equal(t, v.Group, status.HighPriorityPlan[i].Group) + if v.Reason != "*" { + assert.Equal(t, v.Reason, status.HighPriorityPlan[i].Reason) + } + } + } + require.Len(t, status.Plan, len(testCase.ExpectedPlan)) for i, v := range testCase.ExpectedPlan { assert.Equal(t, v.Type, status.Plan[i].Type) assert.Equal(t, v.Group, status.Plan[i].Group) + if v.Reason != "*" { + assert.Equal(t, v.Reason, status.Plan[i].Reason) + } } }) } diff --git a/pkg/deployment/reconcile/plan_builder_tls.go b/pkg/deployment/reconcile/plan_builder_tls.go index 7d8eed114..386d69036 100644 --- a/pkg/deployment/reconcile/plan_builder_tls.go +++ b/pkg/deployment/reconcile/plan_builder_tls.go @@ -308,12 +308,9 @@ func createKeyfileRenewalPlanDefault(ctx context.Context, lCtx, c := context.WithTimeout(ctx, 500*time.Millisecond) defer c() - if renew, recreate := keyfileRenewalRequired(lCtx, log, apiObject, spec, cachedStatus, planCtx, group, member, api.TLSRotateModeRecreate); renew { + if renew, _ := keyfileRenewalRequired(lCtx, log, apiObject, spec, cachedStatus, planCtx, group, member, api.TLSRotateModeRecreate); renew { log.Info().Msg("Renewal of keyfile required - Recreate") - if recreate { - plan = append(plan, api.NewAction(api.ActionTypeCleanTLSKeyfileCertificate, group, member.ID, "Remove server keyfile and enforce renewal")) - } - plan = append(plan, createRotateMemberPlan(log, member, group, "Restart server after keyfile removal")...) + plan = append(plan, tlsRotateConditionPlan(group, member.ID, "Restart server after keyfile removal")...) } } diff --git a/pkg/deployment/reconcile/plan_builder_tls_sni.go b/pkg/deployment/reconcile/plan_builder_tls_sni.go index 178681f79..dab65e9ae 100644 --- a/pkg/deployment/reconcile/plan_builder_tls_sni.go +++ b/pkg/deployment/reconcile/plan_builder_tls_sni.go @@ -108,7 +108,7 @@ func createRotateTLSServerSNIPlan(ctx context.Context, } else if !ok { switch spec.TLS.Mode.Get() { case api.TLSRotateModeRecreate: - plan = append(plan, createRotateMemberPlan(log, m, group, "SNI Secret needs update")...) + plan = append(plan, tlsRotateConditionPlan(group, m.ID, "SNI Secret needs update")...) case api.TLSRotateModeInPlace: plan = append(plan, api.NewAction(api.ActionTypeUpdateTLSSNI, group, m.ID, "SNI Secret needs update")) diff --git a/pkg/deployment/reconcile/plan_builder_update.go b/pkg/deployment/reconcile/plan_builder_update.go new file mode 100644 index 000000000..3f58a9819 --- /dev/null +++ b/pkg/deployment/reconcile/plan_builder_update.go @@ -0,0 +1,23 @@ +// +// DISCLAIMER +// +// Copyright 2021 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile diff --git a/pkg/deployment/reconcile/plan_builder_utils.go b/pkg/deployment/reconcile/plan_builder_utils.go index e8aa62e3e..94dc2cc16 100644 --- a/pkg/deployment/reconcile/plan_builder_utils.go +++ b/pkg/deployment/reconcile/plan_builder_utils.go @@ -29,8 +29,6 @@ import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/agency" "github.com/arangodb/kube-arangodb/pkg/util/errors" - "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" - inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" "github.com/rs/zerolog" ) @@ -66,65 +64,3 @@ func fetchAgency(ctx context.Context, spec api.DeploymentSpec, status api.Deploy return nil, errors.Newf("not able to read from agency when agency is down") } } - -type planBuilder func(ctx context.Context, - log zerolog.Logger, apiObject k8sutil.APIObject, - spec api.DeploymentSpec, status api.DeploymentStatus, - cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan - -type planBuilderCondition func(ctx context.Context, - log zerolog.Logger, apiObject k8sutil.APIObject, - spec api.DeploymentSpec, status api.DeploymentStatus, - cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) bool - -type planBuilderSubPlan func(ctx context.Context, - log zerolog.Logger, apiObject k8sutil.APIObject, - spec api.DeploymentSpec, status api.DeploymentStatus, - cachedStatus inspectorInterface.Inspector, context PlanBuilderContext, w WithPlanBuilder, plans ...planBuilder) api.Plan - -func NewWithPlanBuilder(ctx context.Context, - log zerolog.Logger, apiObject k8sutil.APIObject, - spec api.DeploymentSpec, status api.DeploymentStatus, - cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) WithPlanBuilder { - return &withPlanBuilder{ - ctx: ctx, - log: log, - apiObject: apiObject, - spec: spec, - status: status, - cachedStatus: cachedStatus, - context: context, - } -} - -type WithPlanBuilder interface { - Apply(p planBuilder) api.Plan - ApplyWithCondition(c planBuilderCondition, p planBuilder) api.Plan - ApplySubPlan(p planBuilderSubPlan, plans ...planBuilder) api.Plan -} - -type withPlanBuilder struct { - ctx context.Context - log zerolog.Logger - apiObject k8sutil.APIObject - spec api.DeploymentSpec - status api.DeploymentStatus - cachedStatus inspectorInterface.Inspector - context PlanBuilderContext -} - -func (w withPlanBuilder) ApplyWithCondition(c planBuilderCondition, p planBuilder) api.Plan { - if !c(w.ctx, w.log, w.apiObject, w.spec, w.status, w.cachedStatus, w.context) { - return api.Plan{} - } - - return w.Apply(p) -} - -func (w withPlanBuilder) ApplySubPlan(p planBuilderSubPlan, plans ...planBuilder) api.Plan { - return p(w.ctx, w.log, w.apiObject, w.spec, w.status, w.cachedStatus, w.context, w, plans...) -} - -func (w withPlanBuilder) Apply(p planBuilder) api.Plan { - return p(w.ctx, w.log, w.apiObject, w.spec, w.status, w.cachedStatus, w.context) -} diff --git a/pkg/deployment/resilience/member_failure.go b/pkg/deployment/resilience/member_failure.go index 9eb415131..82da4f4b8 100644 --- a/pkg/deployment/resilience/member_failure.go +++ b/pkg/deployment/resilience/member_failure.go @@ -55,7 +55,7 @@ func (r *Resilience) CheckMemberFailure(ctx context.Context) error { // Check if there are Members with Phase Upgrading or Rotation but no plan switch m.Phase { - case api.MemberPhaseNone: + case api.MemberPhaseNone, api.MemberPhasePending: continue case api.MemberPhaseUpgrading, api.MemberPhaseRotating, api.MemberPhaseCleanOut, api.MemberPhaseRotateStart: if len(status.Plan) == 0 { diff --git a/pkg/deployment/resources/context.go b/pkg/deployment/resources/context.go index 9cf17daad..0cd07f50d 100644 --- a/pkg/deployment/resources/context.go +++ b/pkg/deployment/resources/context.go @@ -39,7 +39,7 @@ import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" - v1 "k8s.io/api/core/v1" + core "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) @@ -67,11 +67,41 @@ type DeploymentAgencyMaintenance interface { SetAgencyMaintenanceMode(ctx context.Context, enabled bool) error } +type DeploymentPodRenderer interface { + // RenderPodForMember Renders Pod definition for member + RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) + // RenderPodTemplateForMember Renders PodTemplate definition for member + RenderPodTemplateForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.PodTemplateSpec, error) + // RenderPodTemplateForMember Renders PodTemplate definition for member from current state + RenderPodForMemberFromCurrent(ctx context.Context, cachedStatus inspectorInterface.Inspector, memberID string) (*core.Pod, error) + // RenderPodTemplateForMemberFromCurrent Renders PodTemplate definition for member + RenderPodTemplateForMemberFromCurrent(ctx context.Context, cachedStatus inspectorInterface.Inspector, memberID string) (*core.PodTemplateSpec, error) +} + +type DeploymentImageManager interface { + // SelectImage select currently used image by pod + SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) + // SelectImage select currently used image by pod in member + SelectImageForMember(spec api.DeploymentSpec, status api.DeploymentStatus, member api.MemberStatus) (api.ImageInfo, bool) +} + +type ArangoMemberUpdateFunc func(obj *api.ArangoMember) bool +type ArangoMemberStatusUpdateFunc func(obj *api.ArangoMember, s *api.ArangoMemberStatus) bool + +type ArangoMemberContext interface { + // WithArangoMemberUpdate run action with update of ArangoMember + WithArangoMemberUpdate(ctx context.Context, namespace, name string, action ArangoMemberUpdateFunc) error + // WithArangoMemberStatusUpdate run action with update of ArangoMember Status + WithArangoMemberStatusUpdate(ctx context.Context, namespace, name string, action ArangoMemberStatusUpdateFunc) error +} + // Context provides all functions needed by the Resources service // to perform its service. type Context interface { DeploymentStatusUpdate DeploymentAgencyMaintenance + ArangoMemberContext + DeploymentImageManager // GetAPIObject returns the deployment as k8s object. GetAPIObject() k8sutil.APIObject @@ -106,10 +136,10 @@ type Context interface { // On error, the error is logged. CreateEvent(evt *k8sutil.Event) // GetOwnedPVCs returns a list of all PVCs owned by the deployment. - GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) + GetOwnedPVCs() ([]core.PersistentVolumeClaim, error) // CleanupPod deletes a given pod with force and explicit UID. // If the pod does not exist, the error is ignored. - CleanupPod(ctx context.Context, p *v1.Pod) error + CleanupPod(ctx context.Context, p *core.Pod) error // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. DeletePod(ctx context.Context, podName string) error diff --git a/pkg/deployment/resources/inspector/inspector.go b/pkg/deployment/resources/inspector/inspector.go index d70074ebf..341c0aae7 100644 --- a/pkg/deployment/resources/inspector/inspector.go +++ b/pkg/deployment/resources/inspector/inspector.go @@ -27,6 +27,8 @@ import ( "context" "sync" + "github.com/arangodb/kube-arangodb/pkg/util/errors" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" @@ -46,49 +48,19 @@ type SecretReadInterface interface { Get(ctx context.Context, name string, opts meta.GetOptions) (*core.Secret, error) } -func NewInspector(k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, c versioned.Interface, namespace string) (inspectorInterface.Inspector, error) { - ctx := context.TODO() - pods, err := podsToMap(ctx, k, namespace) - if err != nil { +func NewInspector(ctx context.Context, k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, c versioned.Interface, namespace string) (inspectorInterface.Inspector, error) { + i := &inspector{ + namespace: namespace, + k: k, + m: m, + c: c, + } + + if err := i.Refresh(ctx); err != nil { return nil, err } - secrets, err := secretsToMap(ctx, k, namespace) - if err != nil { - return nil, err - } - - pvcs, err := pvcsToMap(ctx, k, namespace) - if err != nil { - return nil, err - } - - services, err := servicesToMap(ctx, k, namespace) - if err != nil { - return nil, err - } - - serviceAccounts, err := serviceAccountsToMap(ctx, k, namespace) - if err != nil { - return nil, err - } - - podDisruptionBudgets, err := podDisruptionBudgetsToMap(ctx, k, namespace) - if err != nil { - return nil, err - } - - serviceMonitors, err := serviceMonitorsToMap(ctx, m, namespace) - if err != nil { - return nil, err - } - - arangoMembers, err := arangoMembersToMap(ctx, c, namespace) - if err != nil { - return nil, err - } - - return NewInspectorFromData(pods, secrets, pvcs, services, serviceAccounts, podDisruptionBudgets, serviceMonitors, arangoMembers), nil + return i, nil } func NewEmptyInspector() inspectorInterface.Inspector { @@ -118,6 +90,12 @@ func NewInspectorFromData(pods map[string]*core.Pod, type inspector struct { lock sync.Mutex + namespace string + + k kubernetes.Interface + m monitoringClient.MonitoringV1Interface + c versioned.Interface + pods map[string]*core.Pod secrets map[string]*core.Secret pvcs map[string]*core.PersistentVolumeClaim @@ -128,47 +106,50 @@ type inspector struct { arangoMembers map[string]*api.ArangoMember } -func (i *inspector) Refresh(ctx context.Context, k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, - c versioned.Interface, namespace string) error { +func (i *inspector) Refresh(ctx context.Context) error { i.lock.Lock() defer i.lock.Unlock() - pods, err := podsToMap(ctx, k, namespace) + if i.namespace == "" { + return errors.New("Inspector created fro mstatic data") + } + + pods, err := podsToMap(ctx, i.k, i.namespace) if err != nil { return err } - secrets, err := secretsToMap(ctx, k, namespace) + secrets, err := secretsToMap(ctx, i.k, i.namespace) if err != nil { return err } - pvcs, err := pvcsToMap(ctx, k, namespace) + pvcs, err := pvcsToMap(ctx, i.k, i.namespace) if err != nil { return err } - services, err := servicesToMap(ctx, k, namespace) + services, err := servicesToMap(ctx, i.k, i.namespace) if err != nil { return err } - serviceAccounts, err := serviceAccountsToMap(ctx, k, namespace) + serviceAccounts, err := serviceAccountsToMap(ctx, i.k, i.namespace) if err != nil { return err } - podDisruptionBudgets, err := podDisruptionBudgetsToMap(ctx, k, namespace) + podDisruptionBudgets, err := podDisruptionBudgetsToMap(ctx, i.k, i.namespace) if err != nil { return err } - serviceMonitors, err := serviceMonitorsToMap(ctx, m, namespace) + serviceMonitors, err := serviceMonitorsToMap(ctx, i.m, i.namespace) if err != nil { return err } - arangoMembers, err := arangoMembersToMap(ctx, c, namespace) + arangoMembers, err := arangoMembersToMap(ctx, i.c, i.namespace) if err != nil { return err } diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index ec146bca1..6a6ef4388 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -55,7 +55,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" core "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -187,7 +187,7 @@ func createArangodArgs(cachedStatus interfaces.Inspector, input pod.Input, addit } // createArangoSyncArgs creates command line arguments for an arangosync server in the given group. -func createArangoSyncArgs(apiObject metav1.Object, spec api.DeploymentSpec, group api.ServerGroup, groupSpec api.ServerGroupSpec, member api.MemberStatus) []string { +func createArangoSyncArgs(apiObject meta.Object, spec api.DeploymentSpec, group api.ServerGroup, groupSpec api.ServerGroupSpec, member api.MemberStatus) []string { options := k8sutil.CreateOptionPairs(64) var runCmd string var port int @@ -303,6 +303,45 @@ func (r *Resources) CreatePodTolerations(group api.ServerGroup, groupSpec api.Se return tolerations } +func (r *Resources) RenderPodTemplateForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.PodTemplateSpec, error) { + if p, err := r.RenderPodForMember(ctx, cachedStatus, spec, status, memberID, imageInfo); err != nil { + return nil, err + } else { + return &core.PodTemplateSpec{ + ObjectMeta: p.ObjectMeta, + Spec: p.Spec, + }, nil + } +} + +func (r *Resources) RenderPodTemplateForMemberFromCurrent(ctx context.Context, cachedStatus inspectorInterface.Inspector, memberID string) (*core.PodTemplateSpec, error) { + if p, err := r.RenderPodForMemberFromCurrent(ctx, cachedStatus, memberID); err != nil { + return nil, err + } else { + return &core.PodTemplateSpec{ + ObjectMeta: p.ObjectMeta, + Spec: p.Spec, + }, nil + } +} + +func (r *Resources) RenderPodForMemberFromCurrent(ctx context.Context, cachedStatus inspectorInterface.Inspector, memberID string) (*core.Pod, error) { + spec := r.context.GetSpec() + status, _ := r.context.GetStatus() + + member, _, ok := status.Members.ElementByID(memberID) + if !ok { + return nil, errors.Newf("Member not found") + } + + imageInfo, imageFound := r.SelectImageForMember(spec, status, member) + if !imageFound { + return nil, errors.Newf("ImageInfo not found") + } + + return r.RenderPodForMember(ctx, cachedStatus, spec, status, member.ID, imageInfo) +} + func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) { log := r.log apiObject := r.context.GetAPIObject() @@ -456,8 +495,16 @@ func (r *Resources) SelectImage(spec api.DeploymentSpec, status api.DeploymentSt return imageInfo, true } +func (r *Resources) SelectImageForMember(spec api.DeploymentSpec, status api.DeploymentStatus, member api.MemberStatus) (api.ImageInfo, bool) { + if member.Image != nil { + return *member.Image, true + } + + return r.SelectImage(spec, status) +} + // createPodForMember creates all Pods listed in member status -func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentSpec, memberID string, imageNotFoundOnce *sync.Once, cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentSpec, member *api.ArangoMember, memberID string, imageNotFoundOnce *sync.Once) error { log := r.log status, lastVersion := r.context.GetStatus() @@ -470,6 +517,13 @@ func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentS return nil } + template := member.Status.Template + + if template == nil { + // Template not yet propagated + return errors.Newf("Template not yet propagated") + } + if status.CurrentImage == nil { status.CurrentImage = &imageInfo } @@ -488,24 +542,6 @@ func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentS kubecli := r.context.GetKubeCli() apiObject := r.context.GetAPIObject() - endpoint, err := pod.GenerateMemberEndpoint(cachedStatus, apiObject, spec, group, m) - if err != nil { - return errors.WithStack(err) - } - - if m.Endpoint == nil || *m.Endpoint != endpoint { - // Update endpoint - m.Endpoint = &endpoint - if err := status.Members.Update(m, group); err != nil { - return errors.WithStack(err) - } - } - - pod, err := r.RenderPodForMember(ctx, cachedStatus, spec, status, memberID, imageInfo) - if err != nil { - return errors.WithStack(err) - } - ns := r.context.GetNamespace() secrets := kubecli.CoreV1().Secrets(ns) if !found { @@ -515,9 +551,8 @@ func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentS // Update pod name role := group.AsRole() - roleAbbr := group.AsRoleAbbreviated() - m.PodName = k8sutil.CreatePodName(apiObject.GetName(), roleAbbr, m.ID, CreatePodSuffix(spec)) + m.PodName = template.PodSpec.GetName() newPhase := api.MemberPhaseCreated // Create pod if group.IsArangod() { @@ -527,20 +562,15 @@ func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentS newPhase = api.MemberPhaseUpgrading } - sha, err := ChecksumArangoPod(groupSpec, pod) - if err != nil { - return errors.WithStack(err) - } - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) defer cancel() - uid, err := CreateArangoPod(ctxChild, kubecli, apiObject, spec, group, pod) + uid, err := CreateArangoPod(ctxChild, kubecli, apiObject, spec, group, CreatePodFromTemplate(template.PodSpec)) if err != nil { return errors.WithStack(err) } m.PodUID = uid - m.PodSpecVersion = sha + m.PodSpecVersion = template.PodSpecChecksum m.ArangoVersion = m.Image.ArangoDBVersion m.ImageID = m.Image.ImageID @@ -579,22 +609,16 @@ func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentS } } - sha, err := ChecksumArangoPod(groupSpec, pod) - if err != nil { - return errors.WithStack(err) - } - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) defer cancel() - uid, err := CreateArangoPod(ctxChild, kubecli, apiObject, spec, group, pod) + uid, err := CreateArangoPod(ctxChild, kubecli, apiObject, spec, group, CreatePodFromTemplate(template.PodSpec)) if err != nil { return errors.WithStack(err) } log.Debug().Str("pod-name", m.PodName).Msg("Created pod") m.PodUID = uid - m.Endpoint = &endpoint - m.PodSpecVersion = sha + m.PodSpecVersion = template.PodSpecChecksum } // Record new member phase m.Phase = newPhase @@ -604,7 +628,10 @@ func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentS m.Conditions.Remove(api.ConditionTypeAgentRecoveryNeeded) m.Conditions.Remove(api.ConditionTypeAutoUpgrade) m.Conditions.Remove(api.ConditionTypeUpgradeFailed) + m.Conditions.Remove(api.ConditionTypePendingTLSRotation) + m.Conditions.Remove(api.ConditionTypePendingRestart) m.Upgrade = false + r.log.Info().Str("DEBUG", "10101").Str("pod", m.PodName).Msgf("Updating member") if err := status.Members.Update(m, group); err != nil { return errors.WithStack(err) } @@ -685,6 +712,13 @@ func CreateArangoPod(ctx context.Context, kubecli kubernetes.Interface, deployme return uid, nil } +func CreatePodFromTemplate(p *core.PodTemplateSpec) *core.Pod { + return &core.Pod{ + ObjectMeta: p.ObjectMeta, + Spec: p.Spec, + } +} + func ChecksumArangoPod(groupSpec api.ServerGroupSpec, pod *core.Pod) (string, error) { shaPod := pod.DeepCopy() switch groupSpec.InitContainers.GetMode().Get() { @@ -708,7 +742,7 @@ func (r *Resources) EnsurePods(ctx context.Context, cachedStatus inspectorInterf deploymentStatus, _ := r.context.GetStatus() imageNotFoundOnce := &sync.Once{} - createPodMember := func(group api.ServerGroup, groupSpec api.ServerGroupSpec, status *api.MemberStatusList) error { + if err := iterator.ForeachServerGroup(func(group api.ServerGroup, groupSpec api.ServerGroupSpec, status *api.MemberStatusList) error { for _, m := range *status { if m.Phase != api.MemberPhasePending { continue @@ -716,15 +750,29 @@ func (r *Resources) EnsurePods(ctx context.Context, cachedStatus inspectorInterf if m.Conditions.IsTrue(api.ConditionTypeCleanedOut) { continue } + + member, ok := cachedStatus.ArangoMember(m.ArangoMemberName(r.context.GetName(), group)) + if !ok { + // ArangoMember not found, skip + continue + } + + if member.Status.Template == nil { + r.log.Warn().Msgf("Missing Template") + // Template is missing, nothing to do + continue + } + + r.log.Warn().Msgf("Ensuring pod") + spec := r.context.GetSpec() - if err := r.createPodForMember(ctx, spec, m.ID, imageNotFoundOnce, cachedStatus); err != nil { + if err := r.createPodForMember(ctx, spec, member, m.ID, imageNotFoundOnce); err != nil { + r.log.Warn().Err(err).Msgf("Ensuring pod failed") return errors.WithStack(err) } } return nil - } - - if err := iterator.ForeachServerGroup(createPodMember, &deploymentStatus); err != nil { + }, &deploymentStatus); err != nil { return errors.WithStack(err) } diff --git a/pkg/deployment/resources/pod_inspector.go b/pkg/deployment/resources/pod_inspector.go index 2d2351d08..f0930385c 100644 --- a/pkg/deployment/resources/pod_inspector.go +++ b/pkg/deployment/resources/pod_inspector.go @@ -247,7 +247,7 @@ func (r *Resources) InspectPods(ctx context.Context, cachedStatus inspectorInter if _, exists := cachedStatus.Pod(podName); !exists { log.Debug().Str("pod-name", podName).Msg("Does not exist") switch m.Phase { - case api.MemberPhaseNone: + case api.MemberPhaseNone, api.MemberPhasePending: // Do nothing log.Debug().Str("pod-name", podName).Msg("PodPhase is None, waiting for the pod to be recreated") case api.MemberPhaseShuttingDown, api.MemberPhaseUpgrading, api.MemberPhaseFailed, api.MemberPhaseRotateStart, api.MemberPhaseRotating: diff --git a/pkg/util/k8sutil/inspector/inspector.go b/pkg/util/k8sutil/inspector/inspector.go index 5f11f3fe7..f187dbcb8 100644 --- a/pkg/util/k8sutil/inspector/inspector.go +++ b/pkg/util/k8sutil/inspector/inspector.go @@ -26,7 +26,6 @@ package inspector import ( "context" - "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/arangomember" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/persistentvolumeclaim" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/pod" @@ -35,13 +34,10 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/service" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/serviceaccount" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/servicemonitor" - monitoringClient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/typed/monitoring/v1" - "k8s.io/client-go/kubernetes" ) type Inspector interface { - Refresh(ctx context.Context, k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, - c versioned.Interface, namespace string) error + Refresh(ctx context.Context) error pod.Inspector secret.Inspector