[Feature] Propagation modes (#779)

CHANGELOG.md

@@ -7,6 +7,7 @@
 - Add Ephemeral Volumes for apps feature
 - Check if the DB server is cleaned out.
 - Render Pod Template in ArangoMember Spec and Status
+- Add Pod PropagationModes
 
 ## [1.2.1](https://github.com/arangodb/kube-arangodb/tree/1.2.1) (2021-07-28)
 - Fix ArangoMember race with multiple ArangoDeployments within single namespace

pkg/apis/deployment/v1/conditions.go

@@ -73,6 +73,8 @@ const (
 	ConditionTypeMaintenanceMode ConditionType = "MaintenanceMode"
 	// ConditionTypePendingRestart indicates that restart is required
 	ConditionTypePendingRestart ConditionType = "PendingRestart"
+	// ConditionTypeRestart indicates that restart will be started
+	ConditionTypeRestart ConditionType = "Restart"
 	// ConditionTypePendingTLSRotation indicates that TLS rotation is pending
 	ConditionTypePendingTLSRotation ConditionType = "PendingTLSRotation"
 )

pkg/apis/deployment/v1/deployment_member_propagation_mode.go

@@ -0,0 +1,62 @@
+//
+// DISCLAIMER
+//
+// Copyright 2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package v1
+
+type DeploymentMemberPropagationMode string
+
+func (d *DeploymentMemberPropagationMode) Get() DeploymentMemberPropagationMode {
+	if d == nil {
+		return DeploymentMemberPropagationModeDefault
+	}
+
+	return *d
+}
+
+func (d DeploymentMemberPropagationMode) New() *DeploymentMemberPropagationMode {
+	return &d
+}
+
+func (d DeploymentMemberPropagationMode) String() string {
+	return string(d)
+}
+
+func (d *DeploymentMemberPropagationMode) Equal(b *DeploymentMemberPropagationMode) bool {
+	if d == nil && b == nil {
+		return true
+	}
+
+	if d == nil || b == nil {
+		return false
+	}
+
+	return *d == *b
+}
+
+const (
+	// DeploymentMemberPropagationModeDefault Define default propagation mode
+	DeploymentMemberPropagationModeDefault = DeploymentMemberPropagationModeAlways
+	// DeploymentMemberPropagationModeAlways define mode which restart member whenever change in pod is discovered
+	DeploymentMemberPropagationModeAlways DeploymentMemberPropagationMode = "always"
+	// DeploymentMemberPropagationModeOnRestart propagate member spec whenever pod is restarted. Do not restart member by default
+	DeploymentMemberPropagationModeOnRestart DeploymentMemberPropagationMode = "on-restart"
+)

pkg/apis/deployment/v1/deployment_spec.go

@@ -149,6 +149,8 @@ type DeploymentSpec struct {
 	SyncMasters  ServerGroupSpec `json:"syncmasters"`
 	SyncWorkers  ServerGroupSpec `json:"syncworkers"`
 
+	MemberPropagationMode *DeploymentMemberPropagationMode `json:"memberPropagationMode,omitempty"`
+
 	Chaos ChaosSpec `json:"chaos"`
 
 	Recovery *ArangoDeploymentRecoverySpec `json:"recovery,omitempty"`

pkg/apis/deployment/v1/zz_generated.deepcopy.go

@@ -583,6 +583,11 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	in.Coordinators.DeepCopyInto(&out.Coordinators)
 	in.SyncMasters.DeepCopyInto(&out.SyncMasters)
 	in.SyncWorkers.DeepCopyInto(&out.SyncWorkers)
+	if in.MemberPropagationMode != nil {
+		in, out := &in.MemberPropagationMode, &out.MemberPropagationMode
+		*out = new(DeploymentMemberPropagationMode)
+		**out = **in
+	}
 	in.Chaos.DeepCopyInto(&out.Chaos)
 	if in.Recovery != nil {
 		in, out := &in.Recovery, &out.Recovery

pkg/apis/deployment/v2alpha1/conditions.go

@@ -73,6 +73,8 @@ const (
 	ConditionTypeMaintenanceMode ConditionType = "MaintenanceMode"
 	// ConditionTypePendingRestart indicates that restart is required
 	ConditionTypePendingRestart ConditionType = "PendingRestart"
+	// ConditionTypeRestart indicates that restart will be started
+	ConditionTypeRestart ConditionType = "Restart"
 	// ConditionTypePendingTLSRotation indicates that TLS rotation is pending
 	ConditionTypePendingTLSRotation ConditionType = "PendingTLSRotation"
 )

pkg/apis/deployment/v2alpha1/deployment_member_propagation_mode.go

@@ -0,0 +1,62 @@
+//
+// DISCLAIMER
+//
+// Copyright 2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package v2alpha1
+
+type DeploymentMemberPropagationMode string
+
+func (d *DeploymentMemberPropagationMode) Get() DeploymentMemberPropagationMode {
+	if d == nil {
+		return DeploymentMemberPropagationModeDefault
+	}
+
+	return *d
+}
+
+func (d DeploymentMemberPropagationMode) New() *DeploymentMemberPropagationMode {
+	return &d
+}
+
+func (d DeploymentMemberPropagationMode) String() string {
+	return string(d)
+}
+
+func (d *DeploymentMemberPropagationMode) Equal(b *DeploymentMemberPropagationMode) bool {
+	if d == nil && b == nil {
+		return true
+	}
+
+	if d == nil || b == nil {
+		return false
+	}
+
+	return *d == *b
+}
+
+const (
+	// DeploymentMemberPropagationModeDefault Define default propagation mode
+	DeploymentMemberPropagationModeDefault = DeploymentMemberPropagationModeAlways
+	// DeploymentMemberPropagationModeAlways define mode which restart member whenever change in pod is discovered
+	DeploymentMemberPropagationModeAlways DeploymentMemberPropagationMode = "always"
+	// DeploymentMemberPropagationModeOnRestart propagate member spec whenever pod is restarted. Do not restart member by default
+	DeploymentMemberPropagationModeOnRestart DeploymentMemberPropagationMode = "on-restart"
+)

pkg/apis/deployment/v2alpha1/deployment_spec.go

@@ -149,6 +149,8 @@ type DeploymentSpec struct {
 	SyncMasters  ServerGroupSpec `json:"syncmasters"`
 	SyncWorkers  ServerGroupSpec `json:"syncworkers"`
 
+	MemberPropagationMode *DeploymentMemberPropagationMode `json:"memberPropagationMode,omitempty"`
+
 	Chaos ChaosSpec `json:"chaos"`
 
 	Recovery *ArangoDeploymentRecoverySpec `json:"recovery,omitempty"`

pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go

@@ -583,6 +583,11 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	in.Coordinators.DeepCopyInto(&out.Coordinators)
 	in.SyncMasters.DeepCopyInto(&out.SyncMasters)
 	in.SyncWorkers.DeepCopyInto(&out.SyncWorkers)
+	if in.MemberPropagationMode != nil {
+		in, out := &in.MemberPropagationMode, &out.MemberPropagationMode
+		*out = new(DeploymentMemberPropagationMode)
+		**out = **in
+	}
 	in.Chaos.DeepCopyInto(&out.Chaos)
 	if in.Recovery != nil {
 		in, out := &in.Recovery, &out.Recovery

pkg/deployment/deployment_inspector.go

@@ -303,15 +303,17 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva
 
 		return minInspectionInterval, nil
 	} else if status.AppliedVersion == checksum {
-		if !d.apiObject.Status.IsPlanEmpty() && status.Conditions.IsTrue(api.ConditionTypeUpToDate) {
-			if err = d.updateCondition(ctx, api.ConditionTypeUpToDate, false, "Plan is not empty", "There are pending operations in plan"); err != nil {
+		isUpToDate, reason := d.isUpToDateStatus()
+
+		if !isUpToDate && status.Conditions.IsTrue(api.ConditionTypeUpToDate) {
+			if err = d.updateCondition(ctx, api.ConditionTypeUpToDate, false, reason, "There are pending operations in plan or members are in restart process"); err != nil {
 				return minInspectionInterval, errors.Wrapf(err, "Unable to update UpToDate condition")
 			}
 
 			return minInspectionInterval, nil
 		}
 
-		if d.apiObject.Status.IsPlanEmpty() && !status.Conditions.IsTrue(api.ConditionTypeUpToDate) {
+		if isUpToDate && !status.Conditions.IsTrue(api.ConditionTypeUpToDate) {
 			if err = d.updateCondition(ctx, api.ConditionTypeUpToDate, true, "Spec is Up To Date", "Spec is Up To Date"); err != nil {
 				return minInspectionInterval, errors.Wrapf(err, "Unable to update UpToDate condition")
 			}
@@ -349,6 +351,26 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva
 	return
 }
 
+func (d *Deployment) isUpToDateStatus() (upToDate bool, reason string) {
+	if !d.apiObject.Status.IsPlanEmpty() {
+		return false, "Plan is not empty"
+	}
+
+	upToDate = true
+
+	d.apiObject.Status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error {
+		for _, member := range list {
+			if member.Conditions.IsTrue(api.ConditionTypeRestart) || member.Conditions.IsTrue(api.ConditionTypePendingRestart) {
+				upToDate = false
+				reason = "Pending restarts on members"
+			}
+		}
+		return nil
+	})
+
+	return
+}
+
 func (d *Deployment) refreshMaintenanceTTL(ctx context.Context) {
 	if d.apiObject.Spec.Mode.Get() == api.DeploymentModeSingle {
 		return

pkg/deployment/reconcile/plan_builder_high.go

@@ -25,7 +25,8 @@ package reconcile
 import (
 	"context"
 
-	"github.com/arangodb/kube-arangodb/pkg/apis/deployment"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/rotation"
+
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
@@ -57,6 +58,9 @@ func (d *Reconciler) CreateHighPlan(ctx context.Context, cachedStatus inspectorI
 	for id := len(status.Plan); id < len(newPlan); id++ {
 		action := newPlan[id]
 		d.context.CreateEvent(k8sutil.NewPlanAppendEvent(apiObject, action.Type.String(), action.Group.AsRole(), action.MemberID, action.Reason))
+		if r := action.Reason; r != "" {
+			d.log.Info().Str("Action", action.Type.String()).Str("Role", action.Group.AsRole()).Str("Member", action.MemberID).Str("Type", "High").Msgf(r)
+		}
 	}
 
 	status.HighPriorityPlan = newPlan
@@ -83,7 +87,7 @@ func createHighPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.A
 		ApplyIfEmpty(updateMemberPodTemplateSpec).
 		ApplyIfEmpty(updateMemberPhasePlan).
 		ApplyIfEmpty(createCleanOutPlan).
-		ApplyIfEmpty(updateMemberRotationFlag).
+		ApplyIfEmpty(updateMemberRotationConditionsPlan).
 		Plan(), true
 }
 
@@ -118,11 +122,16 @@ func updateMemberPhasePlan(ctx context.Context,
 	status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error {
 		for _, m := range list {
 			if m.Phase == api.MemberPhaseNone {
-				plan = append(plan,
+				p := api.Plan{
 					api.NewAction(api.ActionTypeMemberRIDUpdate, group, m.ID, "Regenerate member RID"),
-					api.NewAction(api.ActionTypeArangoMemberUpdatePodStatus, group, m.ID, "Propagating status of pod"),
-					api.NewAction(api.ActionTypeMemberPhaseUpdate, group, m.ID,
-						"Move to Pending phase").AddParam(ActionTypeMemberPhaseUpdatePhaseKey, api.MemberPhasePending.String()))
+				}
+
+				p = append(p, api.NewAction(api.ActionTypeArangoMemberUpdatePodStatus, group, m.ID, "Propagating status of pod"))
+
+				p = append(p, api.NewAction(api.ActionTypeMemberPhaseUpdate, group, m.ID,
+					"Move to Pending phase").AddParam(ActionTypeMemberPhaseUpdatePhaseKey, api.MemberPhasePending.String()))
+
+				plan = append(plan, p...)
 			}
 		}
 
@@ -132,66 +141,88 @@ func updateMemberPhasePlan(ctx context.Context,
 	return plan
 }
 
-func updateMemberRotationFlag(ctx context.Context,
+func pendingRestartMemberConditionAction(group api.ServerGroup, memberID string, reason string) api.Action {
+	return api.NewAction(api.ActionTypeSetMemberCondition, group, memberID, reason).AddParam(api.ConditionTypePendingRestart.String(), "T")
+}
+
+func restartMemberConditionAction(group api.ServerGroup, memberID string, reason string) api.Action {
+	return pendingRestartMemberConditionAction(group, memberID, reason).AddParam(api.ConditionTypeRestart.String(), "T")
+}
+
+func tlsRotateConditionAction(group api.ServerGroup, memberID string, reason string) api.Action {
+	return api.NewAction(api.ActionTypeSetMemberCondition, group, memberID, reason).AddParam(api.ConditionTypePendingTLSRotation.String(), "T")
+}
+
+func updateMemberRotationConditionsPlan(ctx context.Context,
 	log zerolog.Logger, apiObject k8sutil.APIObject,
 	spec api.DeploymentSpec, status api.DeploymentStatus,
 	cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan {
 	var plan api.Plan
 
-	status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error {
+	if err := status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error {
 		for _, m := range list {
-			p, found := cachedStatus.Pod(m.PodName)
-			if !found {
-				continue
+			p, ok := cachedStatus.Pod(m.PodName)
+			if !ok {
+				p = nil
 			}
 
-			if required, reason := updateMemberRotationFlagConditionCheck(log, apiObject, spec, cachedStatus, m, group, p); required {
-				log.Info().Msgf(reason)
-				plan = append(plan, restartMemberConditionPlan(group, m.ID, reason)...)
+			if p, err := updateMemberRotationConditions(log, apiObject, spec, cachedStatus, m, group, p); err != nil {
+				return err
+			} else if len(p) > 0 {
+				plan = append(plan, p...)
 			}
 		}
 
 		return nil
-	})
+	}); err != nil {
+		log.Err(err).Msgf("Error while generating rotation plan")
+		return nil
+	}
 
 	return plan
 }
 
-func restartMemberConditionPlan(group api.ServerGroup, memberID string, reason string) api.Plan {
-	return api.Plan{
-		api.NewAction(api.ActionTypeSetMemberCondition, group, memberID, reason).AddParam(api.ConditionTypePendingRestart.String(), "T"),
-	}
-}
-
-func tlsRotateConditionPlan(group api.ServerGroup, memberID string, reason string) api.Plan {
-	return api.Plan{
-		api.NewAction(api.ActionTypeSetMemberCondition, group, memberID, reason).AddParam(api.ConditionTypePendingTLSRotation.String(), "T"),
-	}
-}
-
-func updateMemberRotationFlagConditionCheck(log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, cachedStatus inspectorInterface.Inspector, m api.MemberStatus, group api.ServerGroup, p *core.Pod) (bool, string) {
-	if m.Conditions.IsTrue(api.ConditionTypePendingRestart) {
-		return false, ""
+func updateMemberRotationConditions(log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, cachedStatus inspectorInterface.Inspector, member api.MemberStatus, group api.ServerGroup, p *core.Pod) (api.Plan, error) {
+	if member.Conditions.IsTrue(api.ConditionTypeRestart) {
+		return nil, nil
 	}
 
-	if m.Conditions.IsTrue(api.ConditionTypePendingTLSRotation) {
-		return true, "TLS Rotation pending"
+	arangoMember, ok := cachedStatus.ArangoMember(member.ArangoMemberName(apiObject.GetName(), group))
+	if !ok {
+		return nil, nil
 	}
 
-	pvc, exists := cachedStatus.PersistentVolumeClaim(m.PersistentVolumeClaimName)
-	if exists {
-		if k8sutil.IsPersistentVolumeClaimFileSystemResizePending(pvc) {
-			return true, "PVC Resize pending"
+	if m, _, reason, err := rotation.IsRotationRequired(log, cachedStatus, spec, member, p, arangoMember.Spec.Template, arangoMember.Status.Template); err != nil {
+		log.Error().Err(err).Msgf("Error while getting rotation details")
+		return nil, err
+	} else {
+		switch m {
+		case rotation.EnforcedRotation:
+			if reason != "" {
+				log.Info().Bool("enforced", true).Msgf(reason)
+			} else {
+				log.Info().Bool("enforced", true).Msgf("Unknown reason")
+			}
+			// We need to do enforced rotation
+			return api.Plan{restartMemberConditionAction(group, member.ID, reason)}, nil
+		case rotation.GracefulRotation, rotation.InPlaceRotation, rotation.SilentRotation: // TODO: Add support for InPlace and Silent rotation
+			if reason != "" {
+				log.Info().Bool("enforced", false).Msgf(reason)
+			} else {
+				log.Info().Bool("enforced", false).Msgf("Unknown reason")
+			}
+			// We need to do graceful rotation
+			if member.Conditions.IsTrue(api.ConditionTypePendingRestart) {
+				return nil, nil
+			}
+
+			if spec.MemberPropagationMode.Get() == api.DeploymentMemberPropagationModeAlways {
+				return api.Plan{restartMemberConditionAction(group, member.ID, reason)}, nil
+			} else {
+				return api.Plan{pendingRestartMemberConditionAction(group, member.ID, reason)}, nil
+			}
+		default:
+			return nil, nil
 		}
 	}
-
-	if changed, reason := podNeedsRotation(log, apiObject, p, spec, group, m, cachedStatus); changed {
-		return true, reason
-	}
-
-	if _, ok := p.Annotations[deployment.ArangoDeploymentPodRotateAnnotation]; ok {
-		return true, "Rotation flag present"
-	}
-
-	return false, ""
 }

pkg/deployment/reconcile/plan_builder_normal.go

@@ -55,6 +55,9 @@ func (d *Reconciler) CreateNormalPlan(ctx context.Context, cachedStatus inspecto
 	for id := len(status.Plan); id < len(newPlan); id++ {
 		action := newPlan[id]
 		d.context.CreateEvent(k8sutil.NewPlanAppendEvent(apiObject, action.Type.String(), action.Group.AsRole(), action.MemberID, action.Reason))
+		if r := action.Reason; r != "" {
+			d.log.Info().Str("Action", action.Type.String()).Str("Role", action.Group.AsRole()).Str("Member", action.MemberID).Str("Type", "Normal").Msgf(r)
+		}
 	}
 
 	status.Plan = newPlan

pkg/deployment/reconcile/plan_builder_rotate_upgrade.go

@@ -27,8 +27,6 @@ import (
 
 	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
 
-	"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
-
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
 
 	"github.com/arangodb/go-driver"
@@ -124,7 +122,7 @@ func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.API
 				newPlan = createUpgradeMemberPlan(log, m, group, "Version upgrade", spec, status,
 					!decision.AutoUpgradeNeeded)
 			} else {
-				if m.Conditions.IsTrue(api.ConditionTypePendingRestart) {
+				if m.Conditions.IsTrue(api.ConditionTypeRestart) {
 					newPlan = createRotateMemberPlan(log, m, group, "Restart flag present")
 				}
 			}
@@ -328,55 +326,6 @@ func arangoMemberPodTemplateNeedsUpdate(ctx context.Context, log zerolog.Logger,
 	return "", false
 }
 
-// podNeedsRotation returns true when the specification of the
-// given pod differs from what it should be according to the
-// given deployment spec.
-// When true is returned, a reason for the rotation is already returned.
-func podNeedsRotation(log zerolog.Logger, apiObject k8sutil.APIObject, p *core.Pod, spec api.DeploymentSpec, group api.ServerGroup, m api.MemberStatus, cachedStatus inspectorInterface.Inspector) (bool, string) {
-	if m.PodUID != p.UID {
-		return true, "Pod UID does not match, this pod is not managed by Operator. Recreating"
-	}
-
-	if m.PodSpecVersion == "" {
-		return true, "Pod Spec Version is nil - recreating pod"
-	}
-
-	member, ok := cachedStatus.ArangoMember(m.ArangoMemberName(apiObject.GetName(), group))
-	if !ok {
-		return false, ""
-	}
-
-	if member.Status.Template == nil {
-		return false, ""
-	}
-
-	if member.Status.Template.RotationNeeded(member.Spec.Template) {
-		log.Info().Str("id", m.ID).Str("Before", m.PodSpecVersion).Msgf("Pod needs rotation - templates does not match")
-		return true, "Pod needs rotation - checksum does not match"
-	}
-
-	endpoint, err := pod.GenerateMemberEndpoint(cachedStatus, apiObject, spec, group, m)
-	if err != nil {
-		log.Err(err).Msg("Error while getting pod endpoint")
-		return false, ""
-	}
-
-	if e := m.Endpoint; e == nil {
-		if spec.CommunicationMethod == nil {
-			// TODO: Remove in 1.2.0 release to allow rotation
-			return false, "Pod endpoint is not set and CommunicationMethod is not set, do not recreate"
-		}
-
-		return true, "Communication method has been set - ensure endpoint"
-	} else {
-		if *e != endpoint {
-			return true, "Pod endpoint changed"
-		}
-	}
-
-	return false, ""
-}
-
 // clusterReadyForUpgrade returns true if the cluster is ready for the next update, that is:
 // 	- all shards are in sync
 // 	- all members are ready and fine

pkg/deployment/reconcile/plan_builder_test.go

@@ -300,9 +300,10 @@ func (c *testContext) GetStatus() (api.DeploymentStatus, int32) {
 func addAgentsToStatus(t *testing.T, status *api.DeploymentStatus, count int) {
 	for i := 0; i < count; i++ {
 		require.NoError(t, status.Members.Add(api.MemberStatus{
-			ID:      fmt.Sprintf("AGNT-%d", i),
-			PodName: fmt.Sprintf("agnt-depl-xxx-%d", i),
-			Phase:   api.MemberPhaseCreated,
+			ID:             fmt.Sprintf("AGNT-%d", i),
+			PodName:        fmt.Sprintf("agnt-depl-xxx-%d", i),
+			PodSpecVersion: "random",
+			Phase:          api.MemberPhaseCreated,
 			Conditions: []api.Condition{
 				{
 					Type:   api.ConditionTypeReady,

pkg/deployment/reconcile/plan_builder_tls.go

@@ -310,7 +310,7 @@ func createKeyfileRenewalPlanDefault(ctx context.Context,
 
 			if renew, _ := keyfileRenewalRequired(lCtx, log, apiObject, spec, cachedStatus, planCtx, group, member, api.TLSRotateModeRecreate); renew {
 				log.Info().Msg("Renewal of keyfile required - Recreate")
-				plan = append(plan, tlsRotateConditionPlan(group, member.ID, "Restart server after keyfile removal")...)
+				plan = append(plan, tlsRotateConditionAction(group, member.ID, "Restart server after keyfile removal"))
 			}
 		}
 

pkg/deployment/reconcile/plan_builder_tls_sni.go

@@ -108,7 +108,7 @@ func createRotateTLSServerSNIPlan(ctx context.Context,
 			} else if !ok {
 				switch spec.TLS.Mode.Get() {
 				case api.TLSRotateModeRecreate:
-					plan = append(plan, tlsRotateConditionPlan(group, m.ID, "SNI Secret needs update")...)
+					plan = append(plan, tlsRotateConditionAction(group, m.ID, "SNI Secret needs update"))
 				case api.TLSRotateModeInPlace:
 					plan = append(plan,
 						api.NewAction(api.ActionTypeUpdateTLSSNI, group, m.ID, "SNI Secret needs update"))

pkg/deployment/reconcile/plan_executor.go

@@ -159,7 +159,7 @@ func (d *Reconciler) executePlan(ctx context.Context, cachedStatus inspectorInte
 		}
 
 		if abort {
-			return plan, true, nil
+			return nil, true, nil
 		}
 
 		if done {

pkg/deployment/resources/pod_creator.go

@@ -620,6 +620,7 @@ func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentS
 		m.PodUID = uid
 		m.PodSpecVersion = template.PodSpecChecksum
 	}
+
 	// Record new member phase
 	m.Phase = newPhase
 	m.Conditions.Remove(api.ConditionTypeReady)
@@ -630,8 +631,10 @@ func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentS
 	m.Conditions.Remove(api.ConditionTypeUpgradeFailed)
 	m.Conditions.Remove(api.ConditionTypePendingTLSRotation)
 	m.Conditions.Remove(api.ConditionTypePendingRestart)
+	m.Conditions.Remove(api.ConditionTypeRestart)
+
 	m.Upgrade = false
-	r.log.Info().Str("DEBUG", "10101").Str("pod", m.PodName).Msgf("Updating member")
+	r.log.Info().Str("pod", m.PodName).Msgf("Updating member")
 	if err := status.Members.Update(m, group); err != nil {
 		return errors.WithStack(err)
 	}

pkg/deployment/rotation/check.go

@@ -0,0 +1,119 @@
+//
+// DISCLAIMER
+//
+// Copyright 2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package rotation
+
+import (
+	"github.com/arangodb/kube-arangodb/pkg/apis/deployment"
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector"
+	"github.com/rs/zerolog"
+	core "k8s.io/api/core/v1"
+)
+
+type Mode int
+
+const (
+	EnforcedRotation Mode = iota
+	GracefulRotation
+	InPlaceRotation
+	SilentRotation
+	SkippedRotation
+)
+
+func (m Mode) And(b Mode) Mode {
+	if m < b {
+		return m
+	}
+
+	return b
+}
+
+func IsRotationRequired(log zerolog.Logger, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, member api.MemberStatus, pod *core.Pod, specTemplate, statusTemplate *api.ArangoMemberPodTemplate) (mode Mode, plan api.Plan, reason string, err error) {
+	// Determine if rotation is required based on plan and actions
+
+	// Set default mode for return value
+	mode = SkippedRotation
+
+	if member.Phase.IsPending() {
+		// Skip rotation when we are not yet created
+		return
+	}
+
+	if spec.MemberPropagationMode.Get() == api.DeploymentMemberPropagationModeAlways && member.Conditions.IsTrue(api.ConditionTypePendingRestart) {
+		reason = "Restart is pending"
+		mode = EnforcedRotation
+		return
+	}
+
+	// Check if pod details are propagated
+	if pod != nil {
+		if member.PodUID != pod.UID {
+			reason = "Pod UID does not match, this pod is not managed by Operator. Recreating"
+			mode = EnforcedRotation
+			return
+		}
+
+		if _, ok := pod.Annotations[deployment.ArangoDeploymentPodRotateAnnotation]; ok {
+			reason = "Recreation enforced by annotation"
+			mode = EnforcedRotation
+			return
+		}
+	}
+
+	if member.PodSpecVersion == "" {
+		reason = "Pod Spec Version is nil - recreating pod"
+		mode = EnforcedRotation
+		return
+	}
+
+	if specTemplate == nil || statusTemplate == nil {
+		// If spec or status is nil rotation is not needed
+		return
+	}
+
+	// Check if any of resize events are in place
+	if member.Conditions.IsTrue(api.ConditionTypePendingTLSRotation) {
+		reason = "TLS Rotation pending"
+		mode = EnforcedRotation
+		return
+	}
+
+	pvc, exists := cachedStatus.PersistentVolumeClaim(member.PersistentVolumeClaimName)
+	if exists {
+		if k8sutil.IsPersistentVolumeClaimFileSystemResizePending(pvc) {
+			reason = "PVC Resize pending"
+			mode = EnforcedRotation
+			return
+		}
+	}
+
+	if statusTemplate.RotationNeeded(specTemplate) {
+		reason = "Pod needs rotation - templates does not match"
+		mode = GracefulRotation
+		log.Info().Str("id", member.ID).Str("Before", member.PodSpecVersion).Msgf(reason)
+		return
+	}
+
+	return
+}