[Feature] Container runtime image update (#787)

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
 - Update UBI Image to 8.4
 - Fix ArangoSync Liveness Prove
+- Allow runtime update of Sidecar images
 
 ## [1.2.2](https://github.com/arangodb/kube-arangodb/tree/1.2.2) (2021-09-09)
 - Update 'github.com/arangodb/arangosync-client' dependency to v0.7.0

pkg/apis/deployment/v1/arango_member_pod_template.go

@@ -36,10 +36,16 @@ func GetArangoMemberPodTemplate(pod *core.PodTemplateSpec, podSpecChecksum strin
 		return nil, err
 	}
 
+	checksum := fmt.Sprintf("%0x", sha256.Sum256(data))
+
+	if podSpecChecksum == "" {
+		podSpecChecksum = checksum
+	}
+
 	return &ArangoMemberPodTemplate{
 		PodSpec:         pod,
 		PodSpecChecksum: podSpecChecksum,
-		Checksum:        fmt.Sprintf("%0x", sha256.Sum256(data)),
+		Checksum:        checksum,
 	}, nil
 }
 
@@ -49,6 +55,13 @@ type ArangoMemberPodTemplate struct {
 	Checksum        string                `json:"checksum,omitempty"`
 }
 
+func (a *ArangoMemberPodTemplate) GetChecksum() string {
+	if a == nil {
+		return ""
+	}
+	return a.Checksum
+}
+
 func (a *ArangoMemberPodTemplate) Equals(b *ArangoMemberPodTemplate) bool {
 	if a == nil && b == nil {
 		return true
@@ -58,7 +71,7 @@ func (a *ArangoMemberPodTemplate) Equals(b *ArangoMemberPodTemplate) bool {
 		return false
 	}
 
-	return a.Checksum == b.Checksum && a.PodSpecChecksum == b.PodSpecChecksum
+	return a.Checksum == b.Checksum
 }
 
 func (a *ArangoMemberPodTemplate) RotationNeeded(b *ArangoMemberPodTemplate) bool {
@@ -70,7 +83,7 @@ func (a *ArangoMemberPodTemplate) RotationNeeded(b *ArangoMemberPodTemplate) boo
 		return true
 	}
 
-	return a.PodSpecChecksum != b.PodSpecChecksum
+	return a.Checksum != b.Checksum
 }
 
 func (a *ArangoMemberPodTemplate) EqualPodSpecChecksum(checksum string) bool {

pkg/apis/deployment/v1/arango_member_status.go

@@ -22,16 +22,8 @@
 
 package v1
 
-const (
-	ArangoMemberConditionPendingRestart ConditionType = "pending-restart"
-)
-
 type ArangoMemberStatus struct {
 	Conditions ConditionList `json:"conditions,omitempty"`
 
 	Template *ArangoMemberPodTemplate `json:"template,omitempty"`
 }
-
-func (a ArangoMemberStatus) IsPendingRestart() bool {
-	return a.Conditions.IsTrue(ArangoMemberConditionPendingRestart)
-}

pkg/apis/deployment/v1/conditions.go

@@ -77,6 +77,12 @@ const (
 	ConditionTypeRestart ConditionType = "Restart"
 	// ConditionTypePendingTLSRotation indicates that TLS rotation is pending
 	ConditionTypePendingTLSRotation ConditionType = "PendingTLSRotation"
+	// ConditionTypePendingUpdate indicates that runtime update is pending
+	ConditionTypePendingUpdate ConditionType = "PendingUpdate"
+	// ConditionTypeUpdating indicates that runtime update is in progress
+	ConditionTypeUpdating ConditionType = "Updating"
+	// ConditionTypeUpdateFailed indicates that runtime update failed
+	ConditionTypeUpdateFailed ConditionType = "UpdateFailed"
 )
 
 // Condition represents one current condition of a deployment or deployment member.

pkg/apis/deployment/v1/plan.go

@@ -165,6 +165,10 @@ const (
 	ActionTypeArangoMemberUpdatePodSpec ActionType = "ArangoMemberUpdatePodSpec"
 	// ActionTypeArangoMemberUpdatePodStatus updates pod spec
 	ActionTypeArangoMemberUpdatePodStatus ActionType = "ArangoMemberUpdatePodStatus"
+
+	// Runtime Updates
+	// ActionTypeRuntimeContainerImageUpdate updates container image in runtime
+	ActionTypeRuntimeContainerImageUpdate ActionType = "RuntimeContainerImageUpdate"
 )
 
 const (
@@ -245,6 +249,29 @@ func NewAction(actionType ActionType, group ServerGroup, memberID string, reason
 	return a
 }
 
+// ActionBuilder allows to generate actions based on predefined group and member id
+type ActionBuilder interface {
+	// NewAction instantiates a new Action.
+	NewAction(actionType ActionType, reason ...string) Action
+}
+
+type actionBuilder struct {
+	group    ServerGroup
+	memberID string
+}
+
+func (a actionBuilder) NewAction(actionType ActionType, reason ...string) Action {
+	return NewAction(actionType, a.group, a.memberID, reason...)
+}
+
+// NewActionBuilder create new action builder with provided group and id
+func NewActionBuilder(group ServerGroup, memberID string) ActionBuilder {
+	return actionBuilder{
+		group:    group,
+		memberID: memberID,
+	}
+}
+
 // SetImage sets the Image field to the given value and returns the modified
 // action.
 func (a Action) SetImage(image string) Action {

pkg/apis/deployment/v1/timeouts.go

@@ -33,9 +33,14 @@ const (
 )
 
 type Timeouts struct {
+	// AddMember action timeout
 	AddMember *Timeout `json:"addMember,omitempty"`
 
+	// MaintenanceGracePeriod action timeout
 	MaintenanceGracePeriod *Timeout `json:"maintenanceGracePeriod,omitempty"`
+
+	// RuntimeContainerImageUpdate action timeout
+	RuntimeContainerImageUpdate *Timeout `json:"runtimeContainerImageUpdate,omitempty"`
 }
 
 func (t *Timeouts) GetMaintenanceGracePeriod() time.Duration {

pkg/apis/deployment/v1/zz_generated.deepcopy.go

@@ -2136,6 +2136,11 @@ func (in *Timeouts) DeepCopyInto(out *Timeouts) {
 		*out = new(Timeout)
 		**out = **in
 	}
+	if in.RuntimeContainerImageUpdate != nil {
+		in, out := &in.RuntimeContainerImageUpdate, &out.RuntimeContainerImageUpdate
+		*out = new(Timeout)
+		**out = **in
+	}
 	return
 }
 

pkg/apis/deployment/v2alpha1/arango_member_pod_template.go

@@ -36,10 +36,16 @@ func GetArangoMemberPodTemplate(pod *core.PodTemplateSpec, podSpecChecksum strin
 		return nil, err
 	}
 
+	checksum := fmt.Sprintf("%0x", sha256.Sum256(data))
+
+	if podSpecChecksum == "" {
+		podSpecChecksum = checksum
+	}
+
 	return &ArangoMemberPodTemplate{
 		PodSpec:         pod,
 		PodSpecChecksum: podSpecChecksum,
-		Checksum:        fmt.Sprintf("%0x", sha256.Sum256(data)),
+		Checksum:        checksum,
 	}, nil
 }
 
@@ -49,6 +55,13 @@ type ArangoMemberPodTemplate struct {
 	Checksum        string                `json:"checksum,omitempty"`
 }
 
+func (a *ArangoMemberPodTemplate) GetChecksum() string {
+	if a == nil {
+		return ""
+	}
+	return a.Checksum
+}
+
 func (a *ArangoMemberPodTemplate) Equals(b *ArangoMemberPodTemplate) bool {
 	if a == nil && b == nil {
 		return true
@@ -58,7 +71,7 @@ func (a *ArangoMemberPodTemplate) Equals(b *ArangoMemberPodTemplate) bool {
 		return false
 	}
 
-	return a.Checksum == b.Checksum && a.PodSpecChecksum == b.PodSpecChecksum
+	return a.Checksum == b.Checksum
 }
 
 func (a *ArangoMemberPodTemplate) RotationNeeded(b *ArangoMemberPodTemplate) bool {
@@ -70,7 +83,7 @@ func (a *ArangoMemberPodTemplate) RotationNeeded(b *ArangoMemberPodTemplate) boo
 		return true
 	}
 
-	return a.PodSpecChecksum != b.PodSpecChecksum
+	return a.Checksum != b.Checksum
 }
 
 func (a *ArangoMemberPodTemplate) EqualPodSpecChecksum(checksum string) bool {

pkg/apis/deployment/v2alpha1/arango_member_status.go

@@ -22,16 +22,8 @@
 
 package v2alpha1
 
-const (
-	ArangoMemberConditionPendingRestart ConditionType = "pending-restart"
-)
-
 type ArangoMemberStatus struct {
 	Conditions ConditionList `json:"conditions,omitempty"`
 
 	Template *ArangoMemberPodTemplate `json:"template,omitempty"`
 }
-
-func (a ArangoMemberStatus) IsPendingRestart() bool {
-	return a.Conditions.IsTrue(ArangoMemberConditionPendingRestart)
-}

pkg/apis/deployment/v2alpha1/conditions.go

@@ -77,6 +77,12 @@ const (
 	ConditionTypeRestart ConditionType = "Restart"
 	// ConditionTypePendingTLSRotation indicates that TLS rotation is pending
 	ConditionTypePendingTLSRotation ConditionType = "PendingTLSRotation"
+	// ConditionTypePendingUpdate indicates that runtime update is pending
+	ConditionTypePendingUpdate ConditionType = "PendingUpdate"
+	// ConditionTypeUpdating indicates that runtime update is in progress
+	ConditionTypeUpdating ConditionType = "Updating"
+	// ConditionTypeUpdateFailed indicates that runtime update failed
+	ConditionTypeUpdateFailed ConditionType = "UpdateFailed"
 )
 
 // Condition represents one current condition of a deployment or deployment member.

pkg/apis/deployment/v2alpha1/plan.go

@@ -165,6 +165,10 @@ const (
 	ActionTypeArangoMemberUpdatePodSpec ActionType = "ArangoMemberUpdatePodSpec"
 	// ActionTypeArangoMemberUpdatePodStatus updates pod spec
 	ActionTypeArangoMemberUpdatePodStatus ActionType = "ArangoMemberUpdatePodStatus"
+
+	// Runtime Updates
+	// ActionTypeRuntimeContainerImageUpdate updates container image in runtime
+	ActionTypeRuntimeContainerImageUpdate ActionType = "RuntimeContainerImageUpdate"
 )
 
 const (
@@ -245,6 +249,29 @@ func NewAction(actionType ActionType, group ServerGroup, memberID string, reason
 	return a
 }
 
+// ActionBuilder allows to generate actions based on predefined group and member id
+type ActionBuilder interface {
+	// NewAction instantiates a new Action.
+	NewAction(actionType ActionType, reason ...string) Action
+}
+
+type actionBuilder struct {
+	group    ServerGroup
+	memberID string
+}
+
+func (a actionBuilder) NewAction(actionType ActionType, reason ...string) Action {
+	return NewAction(actionType, a.group, a.memberID, reason...)
+}
+
+// NewActionBuilder create new action builder with provided group and id
+func NewActionBuilder(group ServerGroup, memberID string) ActionBuilder {
+	return actionBuilder{
+		group:    group,
+		memberID: memberID,
+	}
+}
+
 // SetImage sets the Image field to the given value and returns the modified
 // action.
 func (a Action) SetImage(image string) Action {

pkg/apis/deployment/v2alpha1/timeouts.go

@@ -33,9 +33,14 @@ const (
 )
 
 type Timeouts struct {
+	// AddMember action timeout
 	AddMember *Timeout `json:"addMember,omitempty"`
 
+	// MaintenanceGracePeriod action timeout
 	MaintenanceGracePeriod *Timeout `json:"maintenanceGracePeriod,omitempty"`
+
+	// RuntimeContainerImageUpdate action timeout
+	RuntimeContainerImageUpdate *Timeout `json:"runtimeContainerImageUpdate,omitempty"`
 }
 
 func (t *Timeouts) GetMaintenanceGracePeriod() time.Duration {

pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go

@@ -2136,6 +2136,11 @@ func (in *Timeouts) DeepCopyInto(out *Timeouts) {
 		*out = new(Timeout)
 		**out = **in
 	}
+	if in.RuntimeContainerImageUpdate != nil {
+		in, out := &in.RuntimeContainerImageUpdate, &out.RuntimeContainerImageUpdate
+		*out = new(Timeout)
+		**out = **in
+	}
 	return
 }
 

pkg/deployment/reconcile/action.go

@@ -53,6 +53,22 @@ type Action interface {
 	MemberID() string
 }
 
+// ActionPost keep interface which is executed after action is completed.
+type ActionPost interface {
+	Action
+
+	// Post execute after action is completed
+	Post(ctx context.Context) error
+}
+
+func getActionPost(a Action, ctx context.Context) error {
+	if c, ok := a.(ActionPost); !ok {
+		return nil
+	} else {
+		return c.Post(ctx)
+	}
+}
+
 // ActionReloadCachedStatus keeps information about CachedStatus reloading (executed after action has been executed)
 type ActionReloadCachedStatus interface {
 	Action

pkg/deployment/reconcile/action_arango_membed_update_pod_status.go

@@ -36,6 +36,10 @@ func init() {
 	registerAction(api.ActionTypeArangoMemberUpdatePodStatus, newArangoMemberUpdatePodStatusAction)
 }
 
+const (
+	ActionTypeArangoMemberUpdatePodStatusChecksum = "checksum"
+)
+
 // newArangoMemberUpdatePodStatusAction creates a new Action that implements the given
 // planned ArangoMemberUpdatePodStatus action.
 func newArangoMemberUpdatePodStatusAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
@@ -74,6 +78,17 @@ func (a *actionArangoMemberUpdatePodStatus) Start(ctx context.Context) (bool, er
 		return false, err
 	}
 
+	if c, ok := a.action.GetParam(ActionTypeArangoMemberUpdatePodStatusChecksum); ok {
+		if member.Spec.Template == nil {
+			return true, nil
+		}
+
+		if member.Spec.Template.Checksum != c {
+			// Checksum is invalid
+			return true, nil
+		}
+	}
+
 	if member.Status.Template == nil || !member.Status.Template.Equals(member.Spec.Template) {
 		if err := a.actionCtx.WithArangoMemberStatusUpdate(context.Background(), member.GetNamespace(), member.GetName(), func(obj *api.ArangoMember, status *api.ArangoMemberStatus) bool {
 			if status.Template == nil || !status.Template.Equals(member.Spec.Template) {

pkg/deployment/reconcile/action_context.go

@@ -26,6 +26,10 @@ package reconcile
 import (
 	"context"
 
+	"github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned"
+	monitoringClient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+	"k8s.io/client-go/kubernetes"
+
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
 
 	inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector"
@@ -53,6 +57,7 @@ type ActionContext interface {
 	resources.DeploymentAgencyMaintenance
 	resources.ArangoMemberContext
 	resources.DeploymentPodRenderer
+	resources.DeploymentCLIGetter
 
 	// GetAPIObject returns the deployment as k8s object.
 	GetAPIObject() k8sutil.APIObject
@@ -163,6 +168,18 @@ type actionContext struct {
 	cachedStatus inspectorInterface.Inspector
 }
 
+func (ac *actionContext) GetKubeCli() kubernetes.Interface {
+	return ac.context.GetKubeCli()
+}
+
+func (ac *actionContext) GetMonitoringV1Cli() monitoringClient.MonitoringV1Interface {
+	return ac.context.GetMonitoringV1Cli()
+}
+
+func (ac *actionContext) GetArangoCli() versioned.Interface {
+	return ac.context.GetArangoCli()
+}
+
 func (ac *actionContext) RenderPodForMemberFromCurrent(ctx context.Context, cachedStatus inspectorInterface.Inspector, memberID string) (*core.Pod, error) {
 	return ac.context.RenderPodForMemberFromCurrent(ctx, cachedStatus, memberID)
 }

pkg/deployment/reconcile/action_helper.go

@@ -66,6 +66,10 @@ func newActionImpl(log zerolog.Logger, action api.Action, actionCtx ActionContex
 	return newBaseActionImpl(log, action, actionCtx, NewTimeoutFetcher(timeout), memberIDRef)
 }
 
+func newBaseActionImplDefRef(log zerolog.Logger, action api.Action, actionCtx ActionContext, timeout TimeoutFetcher) actionImpl {
+	return newBaseActionImpl(log, action, actionCtx, timeout, &action.MemberID)
+}
+
 func newBaseActionImpl(log zerolog.Logger, action api.Action, actionCtx ActionContext, timeout TimeoutFetcher, memberIDRef *string) actionImpl {
 	if memberIDRef == nil {
 		panic("Action cannot have nil reference to member!")

pkg/deployment/reconcile/action_runtime_container_image_update.go

@@ -0,0 +1,281 @@
+//
+// DISCLAIMER
+//
+// Copyright 2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+	"time"
+
+	"github.com/arangodb/kube-arangodb/pkg/deployment/rotation"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	"github.com/rs/zerolog"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
+)
+
+func init() {
+	registerAction(api.ActionTypeRuntimeContainerImageUpdate, runtimeContainerImageUpdate)
+}
+
+func runtimeContainerImageUpdate(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &actionRuntimeContainerImageUpdate{}
+
+	a.actionImpl = newBaseActionImplDefRef(log, action, actionCtx, func(deploymentSpec api.DeploymentSpec) time.Duration {
+		return deploymentSpec.Timeouts.Get().AddMember.Get(defaultTimeout)
+	})
+
+	return a
+}
+
+var _ ActionReloadCachedStatus = &actionRuntimeContainerImageUpdate{}
+var _ ActionPost = &actionRuntimeContainerImageUpdate{}
+
+type actionRuntimeContainerImageUpdate struct {
+	// actionImpl implement timeout and member id functions
+	actionImpl
+}
+
+func (a actionRuntimeContainerImageUpdate) Post(ctx context.Context) error {
+	a.log.Info().Msgf("Updating container image")
+	m, ok := a.actionCtx.GetMemberStatusByID(a.action.MemberID)
+	if !ok {
+		a.log.Info().Msg("member is gone already")
+		return nil
+	}
+
+	name, image, ok := a.getContainerDetails()
+	if !ok {
+		a.log.Info().Msg("Unable to find container details")
+		return nil
+	}
+
+	member, ok := a.actionCtx.GetCachedStatus().ArangoMember(m.ArangoMemberName(a.actionCtx.GetName(), a.action.Group))
+	if !ok {
+		err := errors.Newf("ArangoMember not found")
+		a.log.Error().Err(err).Msg("ArangoMember not found")
+		return err
+	}
+
+	return a.actionCtx.WithArangoMemberStatusUpdate(ctx, member.GetNamespace(), member.GetName(), func(obj *api.ArangoMember, s *api.ArangoMemberStatus) bool {
+		if obj.Spec.Template == nil || s.Template == nil ||
+			obj.Spec.Template.PodSpec == nil || s.Template.PodSpec == nil {
+			a.log.Info().Msgf("Nil Member definition")
+			return false
+		}
+
+		if len(obj.Spec.Template.PodSpec.Spec.Containers) != len(s.Template.PodSpec.Spec.Containers) {
+			a.log.Info().Msgf("Invalid size of containers")
+			return false
+		}
+
+		for id := range obj.Spec.Template.PodSpec.Spec.Containers {
+			if obj.Spec.Template.PodSpec.Spec.Containers[id].Name == name {
+				if s.Template.PodSpec.Spec.Containers[id].Name != name {
+					a.log.Info().Msgf("Invalid order of containers")
+					return false
+				}
+
+				if obj.Spec.Template.PodSpec.Spec.Containers[id].Image != image {
+					a.log.Info().Str("got", obj.Spec.Template.PodSpec.Spec.Containers[id].Image).Str("expected", image).Msgf("Invalid spec image of container")
+					return false
+				}
+
+				if s.Template.PodSpec.Spec.Containers[id].Image != image {
+					s.Template.PodSpec.Spec.Containers[id].Image = image
+					return true
+				}
+				return false
+			}
+		}
+		return false
+	})
+}
+
+func (a actionRuntimeContainerImageUpdate) ReloadCachedStatus() bool {
+	return true
+}
+
+func (a actionRuntimeContainerImageUpdate) getContainerDetails() (string, string, bool) {
+	container, ok := a.action.GetParam(rotation.ContainerName)
+	if !ok {
+		return "", "", false
+	}
+
+	image, ok := a.action.GetParam(rotation.ContainerImage)
+	if !ok {
+		return "", "", false
+	}
+
+	return container, image, true
+}
+
+// Start starts the action for changing conditions on the provided member.
+func (a actionRuntimeContainerImageUpdate) Start(ctx context.Context) (bool, error) {
+	m, ok := a.actionCtx.GetMemberStatusByID(a.action.MemberID)
+	if !ok {
+		a.log.Info().Msg("member is gone already")
+		return true, nil
+	}
+
+	name, image, ok := a.getContainerDetails()
+	if !ok {
+		a.log.Info().Msg("Unable to find container details")
+		return true, nil
+	}
+
+	if !m.Phase.IsReady() {
+		a.log.Info().Msg("Member is not ready, unable to run update operation")
+		return true, nil
+	}
+
+	member, ok := a.actionCtx.GetCachedStatus().ArangoMember(m.ArangoMemberName(a.actionCtx.GetName(), a.action.Group))
+	if !ok {
+		err := errors.Newf("ArangoMember not found")
+		a.log.Error().Err(err).Msg("ArangoMember not found")
+		return false, err
+	}
+
+	pod, ok := a.actionCtx.GetCachedStatus().Pod(m.PodName)
+	if !ok {
+		a.log.Info().Msg("pod is not present")
+		return true, nil
+	}
+
+	if member.Spec.Template == nil || member.Spec.Template.PodSpec == nil {
+		a.log.Info().Msg("pod spec is not present")
+		return true, nil
+	}
+
+	if member.Status.Template == nil || member.Status.Template.PodSpec == nil {
+		a.log.Info().Msg("pod status is not present")
+		return true, nil
+	}
+
+	if len(pod.Spec.Containers) != len(member.Spec.Template.PodSpec.Spec.Containers) {
+		a.log.Info().Msg("spec container count is not equal")
+		return true, nil
+	}
+
+	if len(pod.Spec.Containers) != len(member.Status.Template.PodSpec.Spec.Containers) {
+		a.log.Info().Msg("status container count is not equal")
+		return true, nil
+	}
+
+	spec := member.Spec.Template.PodSpec
+	status := member.Status.Template.PodSpec
+
+	for id := range pod.Spec.Containers {
+		if pod.Spec.Containers[id].Name != spec.Spec.Containers[id].Name ||
+			pod.Spec.Containers[id].Name != status.Spec.Containers[id].Name ||
+			pod.Spec.Containers[id].Name != name {
+			continue
+		}
+
+		if pod.Spec.Containers[id].Image != image {
+			// Update pod image
+			pod.Spec.Containers[id].Image = image
+
+			if _, err := a.actionCtx.GetKubeCli().CoreV1().Pods(pod.GetNamespace()).Update(ctx, pod, v1.UpdateOptions{}); err != nil {
+				return true, err
+			}
+
+			// Start wait action
+			return false, nil
+		}
+
+		return true, nil
+	}
+
+	return true, nil
+}
+
+func (a actionRuntimeContainerImageUpdate) CheckProgress(ctx context.Context) (bool, bool, error) {
+
+	a.log.Info().Msgf("Update Progress")
+	m, ok := a.actionCtx.GetMemberStatusByID(a.action.MemberID)
+	if !ok {
+		a.log.Info().Msg("member is gone already")
+		return true, false, nil
+	}
+
+	pod, ok := a.actionCtx.GetCachedStatus().Pod(m.PodName)
+	if !ok {
+		a.log.Info().Msg("pod is not present")
+		return true, false, nil
+	}
+
+	name, image, ok := a.getContainerDetails()
+	if !ok {
+		a.log.Info().Msg("Unable to find container details")
+		return true, false, nil
+	}
+
+	cspec, ok := k8sutil.GetContainerByName(pod, name)
+	if !ok {
+		a.log.Info().Msg("Unable to find container spec")
+		return true, false, nil
+	}
+
+	cstatus, ok := k8sutil.GetContainerStatusByName(pod, name)
+	if !ok {
+		a.log.Info().Msg("Unable to find container status")
+		return true, false, nil
+	}
+
+	if cspec.Image != image {
+		a.log.Info().Msg("Image changed")
+		return true, false, nil
+	}
+
+	if s := cstatus.State.Terminated; s != nil {
+		// We are in terminated state
+		// Image is changed after start
+		if cspec.Image != cstatus.Image {
+			// Image not yet updated, retry soon
+			return false, false, nil
+		}
+
+		// Pod wont get up and running
+		return true, false, errors.Newf("Container %s failed during image replacement: (%d) %s: %s", name, s.ExitCode, s.Reason, s.Message)
+	} else if s := cstatus.State.Waiting; s != nil {
+		// Pod is still pulling image or pending for pod start
+		return false, false, nil
+	} else if s := cstatus.State.Running; s != nil {
+		// Image is changed after restart
+		if cspec.Image != cstatus.Image {
+			// Image not yet updated, retry soon
+			return false, false, nil
+		}
+
+		if k8sutil.IsPodReady(pod) {
+			// Pod is alive again
+			return true, false, nil
+		}
+		return false, false, nil
+	} else {
+		// Unknown state
+		return false, false, nil
+	}
+}

pkg/deployment/reconcile/action_set_member_condition.go

@@ -65,15 +65,21 @@ func (a actionSetMemberCondition) Start(ctx context.Context) (bool, error) {
 	}
 
 	for condition, value := range a.action.Params {
-		set, err := strconv.ParseBool(value)
-		if err != nil {
-			a.log.Error().Err(err).Str("value", value).Msg("can not parse string to boolean")
-			continue
+		if value == "" {
+			a.log.Debug().Msg("remove the condition")
+
+			m.Conditions.Remove(api.ConditionType(condition))
+		} else {
+			set, err := strconv.ParseBool(value)
+			if err != nil {
+				a.log.Error().Err(err).Str("value", value).Msg("can not parse string to boolean")
+				continue
+			}
+
+			a.log.Debug().Msg("set the condition")
+
+			m.Conditions.Update(api.ConditionType(condition), set, a.action.Reason, "action set the member condition")
 		}
-
-		a.log.Debug().Msg("set the condition")
-
-		m.Conditions.Update(api.ConditionType(condition), set, a.action.Reason, "action set the member condition")
 	}
 
 	if err := a.actionCtx.UpdateMember(ctx, m); err != nil {

pkg/deployment/reconcile/context.go

@@ -46,6 +46,7 @@ type Context interface {
 	resources.ArangoMemberContext
 	resources.DeploymentPodRenderer
 	resources.DeploymentImageManager
+	resources.DeploymentCLIGetter
 
 	// GetAPIObject returns the deployment as k8s object.
 	GetAPIObject() k8sutil.APIObject

pkg/deployment/reconcile/plan_builder_high.go

@@ -87,6 +87,7 @@ func createHighPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.A
 		ApplyIfEmpty(updateMemberPodTemplateSpec).
 		ApplyIfEmpty(updateMemberPhasePlan).
 		ApplyIfEmpty(createCleanOutPlan).
+		ApplyIfEmpty(updateMemberUpdateConditionsPlan).
 		ApplyIfEmpty(updateMemberRotationConditionsPlan).
 		Plan(), true
 }
@@ -157,6 +158,38 @@ func tlsRotateConditionAction(group api.ServerGroup, memberID string, reason str
 	return api.NewAction(api.ActionTypeSetMemberCondition, group, memberID, reason).AddParam(api.ConditionTypePendingTLSRotation.String(), "T")
 }
 
+func updateMemberUpdateConditionsPlan(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan {
+	var plan api.Plan
+
+	if err := status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error {
+		for _, m := range list {
+			if m.Conditions.IsTrue(api.ConditionTypeUpdating) {
+				// We are in updating phase
+				if status.Plan.IsEmpty() {
+					// If plan is empty then something went wrong
+					plan = append(plan,
+						api.NewAction(api.ActionTypeSetMemberCondition, group, m.ID, "Clean update actions after failure").
+							AddParam(api.ConditionTypePendingUpdate.String(), "").
+							AddParam(api.ConditionTypeUpdating.String(), "").
+							AddParam(api.ConditionTypeUpdateFailed.String(), "T").
+							AddParam(api.ConditionTypePendingRestart.String(), "T"),
+					)
+				}
+			}
+		}
+
+		return nil
+	}); err != nil {
+		log.Err(err).Msgf("Error while generating update plan")
+		return nil
+	}
+
+	return plan
+}
+
 func updateMemberRotationConditionsPlan(ctx context.Context,
 	log zerolog.Logger, apiObject k8sutil.APIObject,
 	spec api.DeploymentSpec, status api.DeploymentStatus,
@@ -196,7 +229,7 @@ func updateMemberRotationConditions(log zerolog.Logger, apiObject k8sutil.APIObj
 		return nil, nil
 	}
 
-	if m, _, reason, err := rotation.IsRotationRequired(log, cachedStatus, spec, member, p, arangoMember.Spec.Template, arangoMember.Status.Template); err != nil {
+	if m, _, reason, err := rotation.IsRotationRequired(log, cachedStatus, spec, member, group, p, arangoMember.Spec.Template, arangoMember.Status.Template); err != nil {
 		log.Error().Err(err).Msgf("Error while getting rotation details")
 		return nil, err
 	} else {
@@ -209,7 +242,20 @@ func updateMemberRotationConditions(log zerolog.Logger, apiObject k8sutil.APIObj
 			}
 			// We need to do enforced rotation
 			return api.Plan{restartMemberConditionAction(group, member.ID, reason)}, nil
-		case rotation.GracefulRotation, rotation.InPlaceRotation, rotation.SilentRotation: // TODO: Add support for InPlace and Silent rotation
+		case rotation.InPlaceRotation:
+			if member.Conditions.IsTrue(api.ConditionTypeUpdateFailed) {
+				if !(member.Conditions.IsTrue(api.ConditionTypePendingRestart) || member.Conditions.IsTrue(api.ConditionTypeRestart)) {
+					return api.Plan{pendingRestartMemberConditionAction(group, member.ID, reason)}, nil
+				}
+				return nil, nil
+			} else if member.Conditions.IsTrue(api.ConditionTypeUpdating) || member.Conditions.IsTrue(api.ConditionTypePendingUpdate) {
+				return nil, nil
+			}
+			return api.Plan{api.NewAction(api.ActionTypeSetMemberCondition, group, member.ID, reason).AddParam(api.ConditionTypePendingUpdate.String(), "T")}, nil
+		case rotation.SilentRotation:
+			// Propagate changes without restart
+			return api.Plan{api.NewAction(api.ActionTypeArangoMemberUpdatePodStatus, group, member.ID, "Propagating status of pod").AddParam(ActionTypeArangoMemberUpdatePodStatusChecksum, arangoMember.Spec.Template.GetChecksum())}, nil
+		case rotation.GracefulRotation:
 			if reason != "" {
 				log.Info().Bool("enforced", false).Msgf(reason)
 			} else {

pkg/deployment/reconcile/plan_builder_rotate_upgrade.go

@@ -124,8 +124,41 @@ func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.API
 				newPlan = createUpgradeMemberPlan(log, m, group, "Version upgrade", spec, status,
 					!decision.AutoUpgradeNeeded)
 			} else {
-				if rotation.CheckPossible(m) && m.Conditions.IsTrue(api.ConditionTypeRestart) {
-					newPlan = createRotateMemberPlan(log, m, group, "Restart flag present")
+				if rotation.CheckPossible(m) {
+					if m.Conditions.IsTrue(api.ConditionTypeRestart) {
+						newPlan = createRotateMemberPlan(log, m, group, "Restart flag present")
+					} else if m.Conditions.IsTrue(api.ConditionTypeUpdating) || m.Conditions.IsTrue(api.ConditionTypeUpdateFailed) {
+						continue
+					} else if m.Conditions.IsTrue(api.ConditionTypePendingUpdate) {
+						arangoMember, ok := cachedStatus.ArangoMember(m.ArangoMemberName(apiObject.GetName(), group))
+						if !ok {
+							continue
+						}
+						p, ok := cachedStatus.Pod(m.PodName)
+						if !ok {
+							p = nil
+						}
+
+						if mode, p, reason, err := rotation.IsRotationRequired(log, cachedStatus, spec, m, group, p, arangoMember.Spec.Template, arangoMember.Status.Template); err != nil {
+							log.Err(err).Msgf("Error while generating update plan")
+							continue
+						} else if mode != rotation.InPlaceRotation {
+							newPlan = api.Plan{api.NewAction(api.ActionTypeSetMemberCondition, group, m.ID, "Cleaning update").
+								AddParam(api.ConditionTypePendingUpdate.String(), "").AddParam(api.ConditionTypeUpdating.String(), "T")}
+							continue
+						} else {
+							p = p.After(
+								api.NewAction(api.ActionTypeWaitForMemberUp, group, m.ID),
+								api.NewAction(api.ActionTypeWaitForMemberInSync, group, m.ID))
+
+							p = p.Wrap(api.NewAction(api.ActionTypeSetMemberCondition, group, m.ID, reason).
+								AddParam(api.ConditionTypePendingUpdate.String(), "").AddParam(api.ConditionTypeUpdating.String(), "T"),
+								api.NewAction(api.ActionTypeSetMemberCondition, group, m.ID, reason).
+									AddParam(api.ConditionTypeUpdating.String(), ""))
+
+							newPlan = p
+						}
+					}
 				}
 			}
 

pkg/deployment/reconcile/plan_builder_test.go

@@ -29,6 +29,10 @@ import (
 	"io/ioutil"
 	"testing"
 
+	"github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned"
+	monitoringClient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+	"k8s.io/client-go/kubernetes"
+
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
 
 	apiErrors "k8s.io/apimachinery/pkg/api/errors"
@@ -75,6 +79,18 @@ type testContext struct {
 	RecordedEvent    *k8sutil.Event
 }
 
+func (c *testContext) GetKubeCli() kubernetes.Interface {
+	panic("implement me")
+}
+
+func (c *testContext) GetMonitoringV1Cli() monitoringClient.MonitoringV1Interface {
+	panic("implement me")
+}
+
+func (c *testContext) GetArangoCli() versioned.Interface {
+	panic("implement me")
+}
+
 func (c *testContext) RenderPodForMemberFromCurrent(ctx context.Context, cachedStatus inspectorInterface.Inspector, memberID string) (*core.Pod, error) {
 	panic("implement me")
 }

pkg/deployment/reconcile/plan_executor.go

@@ -185,6 +185,11 @@ func (d *Reconciler) executePlan(ctx context.Context, cachedStatus inspectorInte
 				log.Info().Msgf("Appending new plan items")
 				return newPlan, true, nil
 			}
+
+			if err := getActionPost(action, ctx); err != nil {
+				log.Err(err).Msgf("Post action failed")
+				return nil, true, errors.WithStack(err)
+			}
 		} else {
 			if plan[0].StartTime.IsZero() {
 				now := metav1.Now()

pkg/deployment/resources/context.go

@@ -27,11 +27,11 @@ import (
 	"context"
 
 	"github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned"
+	monitoringClient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+	"k8s.io/client-go/kubernetes"
 
 	"github.com/arangodb/kube-arangodb/pkg/operator/scope"
 
-	monitoringClient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
-
 	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
 
 	driver "github.com/arangodb/go-driver"
@@ -40,7 +40,6 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector"
 	core "k8s.io/api/core/v1"
-	"k8s.io/client-go/kubernetes"
 )
 
 // ServerGroupIterator provides a helper to callback on every server
@@ -85,6 +84,15 @@ type DeploymentImageManager interface {
 	SelectImageForMember(spec api.DeploymentSpec, status api.DeploymentStatus, member api.MemberStatus) (api.ImageInfo, bool)
 }
 
+type DeploymentCLIGetter interface {
+	// GetKubeCli returns the kubernetes client
+	GetKubeCli() kubernetes.Interface
+	// GetMonitoringV1Cli returns monitoring client
+	GetMonitoringV1Cli() monitoringClient.MonitoringV1Interface
+	// GetArangoCli returns the Arango CRD client
+	GetArangoCli() versioned.Interface
+}
+
 type ArangoMemberUpdateFunc func(obj *api.ArangoMember) bool
 type ArangoMemberStatusUpdateFunc func(obj *api.ArangoMember, s *api.ArangoMemberStatus) bool
 
@@ -102,6 +110,7 @@ type Context interface {
 	DeploymentAgencyMaintenance
 	ArangoMemberContext
 	DeploymentImageManager
+	DeploymentCLIGetter
 
 	// GetAPIObject returns the deployment as k8s object.
 	GetAPIObject() k8sutil.APIObject
@@ -114,12 +123,6 @@ type Context interface {
 	// UpdateStatus replaces the status of the deployment with the given status and
 	// updates the resources in k8s.
 	UpdateStatus(ctx context.Context, status api.DeploymentStatus, lastVersion int32, force ...bool) error
-	// GetKubeCli returns the kubernetes client
-	GetKubeCli() kubernetes.Interface
-	// GetMonitoringV1Cli returns monitoring client
-	GetMonitoringV1Cli() monitoringClient.MonitoringV1Interface
-	// GetArangoCli returns the Arango CRD client
-	GetArangoCli() versioned.Interface
 	// GetLifecycleImage returns the image name containing the lifecycle helper (== name of operator image)
 	GetLifecycleImage() string
 	// GetOperatorUUIDImage returns the image name containing the uuid helper (== name of operator image)

pkg/deployment/resources/pod_creator.go

@@ -26,7 +26,6 @@ package resources
 import (
 	"context"
 	"crypto/sha1"
-	"crypto/sha256"
 	"encoding/json"
 	"fmt"
 	"net"
@@ -632,6 +631,9 @@ func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentS
 	m.Conditions.Remove(api.ConditionTypePendingTLSRotation)
 	m.Conditions.Remove(api.ConditionTypePendingRestart)
 	m.Conditions.Remove(api.ConditionTypeRestart)
+	m.Conditions.Remove(api.ConditionTypePendingUpdate)
+	m.Conditions.Remove(api.ConditionTypeUpdating)
+	m.Conditions.Remove(api.ConditionTypeUpdateFailed)
 	m.Conditions.Remove(api.ConditionTypeCleanedOut)
 
 	m.Upgrade = false
@@ -737,7 +739,7 @@ func ChecksumArangoPod(groupSpec api.ServerGroupSpec, pod *core.Pod) (string, er
 		return "", err
 	}
 
-	return fmt.Sprintf("%0x", sha256.Sum256(data)), nil
+	return util.SHA256(data), nil
 }
 
 // EnsurePods creates all Pods listed in member status

pkg/deployment/resources/pod_inspector.go

@@ -185,7 +185,7 @@ func (r *Resources) InspectPods(ctx context.Context, cachedStatus inspectorInter
 			}
 		}
 
-		if k8sutil.IsPodReady(pod) {
+		if k8sutil.IsContainerReady(pod, k8sutil.ServerContainerName) {
 			// Pod is now ready
 			if memberStatus.Conditions.Update(api.ConditionTypeReady, true, "Pod Ready", "") {
 				log.Debug().Str("pod-name", pod.GetName()).Msg("Updating member condition Ready to true")

pkg/deployment/rotation/arangod_containers.go

@@ -0,0 +1,102 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package rotation
+
+import (
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	core "k8s.io/api/core/v1"
+)
+
+const (
+	ContainerName  = "name"
+	ContainerImage = "image"
+)
+
+func containersCompare(deploymentSpec api.DeploymentSpec, group api.ServerGroup, spec, status *core.PodSpec) compareFunc {
+	return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
+		a, b := spec.Containers, status.Containers
+
+		if len(a) == 0 || len(b) == 0 {
+			return SkippedRotation, nil, nil
+		}
+
+		for id := range a {
+			if ac, bc := &a[id], &b[id]; ac.Name == k8sutil.ServerContainerName && ac.Name == bc.Name {
+				// Nothing to do
+			} else if ac.Name == bc.Name {
+				if ac.Image != bc.Image {
+					// Image changed
+					plan = append(plan, builder.NewAction(api.ActionTypeRuntimeContainerImageUpdate).AddParam(ContainerName, ac.Name).AddParam(ContainerImage, ac.Image))
+
+					bc.Image = ac.Image
+					mode = mode.And(InPlaceRotation)
+				}
+			}
+		}
+
+		return
+	}
+}
+func initContainersCompare(deploymentSpec api.DeploymentSpec, group api.ServerGroup, spec, status *core.PodSpec) compareFunc {
+	return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
+		gs := deploymentSpec.GetServerGroupSpec(group)
+
+		switch gs.InitContainers.GetMode().Get() {
+		case api.ServerGroupInitContainerIgnoreMode:
+			// Just copy spec to status if different
+			if equal, err := util.CompareJSON(spec.InitContainers, status.InitContainers); err != nil {
+				return 0, nil, err
+			} else if !equal {
+				status.InitContainers = spec.InitContainers
+				mode = mode.And(SilentRotation)
+			} else {
+				return 0, nil, err
+			}
+		default:
+			if len(status.InitContainers) != len(spec.InitContainers) {
+				// Nothing to do, count is different
+				return
+			}
+
+			for id := range status.InitContainers {
+				if status.InitContainers[id].Name != spec.InitContainers[id].Name {
+					// Nothing to do, order is different
+					return
+				}
+			}
+
+			for id := range status.InitContainers {
+				if api.IsReservedServerGroupInitContainerName(status.InitContainers[id].Name) {
+					if equal, err := util.CompareJSON(spec.InitContainers[id], status.InitContainers[id]); err != nil {
+						return 0, nil, err
+					} else if !equal {
+						status.InitContainers[id] = spec.InitContainers[id]
+						mode = mode.And(SilentRotation)
+					}
+				}
+			}
+		}
+
+		return
+	}
+}

pkg/deployment/rotation/arangod_containers_test.go

@@ -0,0 +1,146 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package rotation
+
+import (
+	"testing"
+
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+
+	v1 "k8s.io/api/core/v1"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+)
+
+func Test_ArangoDContainers_SidecarImages(t *testing.T) {
+	testCases := []TestCase{
+		{
+			name:   "Sidecar Image Update",
+			spec:   buildPodSpec(addContainer(k8sutil.ServerContainerName, nil), addSidecarWithImage("sidecar", "local:1.0")),
+			status: buildPodSpec(addContainer(k8sutil.ServerContainerName, nil), addSidecarWithImage("sidecar", "local:2.0")),
+
+			expectedMode: InPlaceRotation,
+			expectedPlan: api.Plan{
+				api.NewAction(api.ActionTypeRuntimeContainerImageUpdate, 0, ""),
+			},
+		},
+		{
+			name:   "Sidecar Image Update with more than one sidecar",
+			spec:   buildPodSpec(addSidecarWithImage("sidecar1", "local:1.0"), addSidecarWithImage("sidecar", "local:1.0")),
+			status: buildPodSpec(addSidecarWithImage("sidecar1", "local:1.0"), addSidecarWithImage("sidecar", "local:2.0")),
+
+			expectedMode: InPlaceRotation,
+			expectedPlan: api.Plan{
+				api.NewAction(api.ActionTypeRuntimeContainerImageUpdate, 0, ""),
+			},
+		},
+	}
+
+	runTestCases(t)(testCases...)
+}
+
+func Test_InitContainers(t *testing.T) {
+	t.Run("Ignore", func(t *testing.T) {
+		testCases := []TestCase{
+			{
+				name: "Same containers",
+				spec: buildPodSpec(addInitContainer(api.ServerGroupReservedInitContainerNameUUID, nil), addInitContainer("sidecar", func(c *v1.Container) {
+					c.Image = "local:1.0"
+				})),
+				status: buildPodSpec(addInitContainer(api.ServerGroupReservedInitContainerNameUUID, nil), addInitContainer("sidecar", func(c *v1.Container) {
+					c.Image = "local:1.0"
+				})),
+
+				expectedMode: SkippedRotation,
+
+				deploymentSpec: buildDeployment(func(depl *api.DeploymentSpec) {
+					depl.Agents.InitContainers = &api.ServerGroupInitContainers{
+						Mode: api.ServerGroupInitContainerIgnoreMode.New(),
+					}
+				}),
+			},
+			{
+				name: "Containers with different image",
+				spec: buildPodSpec(addInitContainer(api.ServerGroupReservedInitContainerNameUUID, nil), addInitContainer("sidecar", func(c *v1.Container) {
+					c.Image = "local:1.0"
+				})),
+				status: buildPodSpec(addInitContainer(api.ServerGroupReservedInitContainerNameUUID, nil), addInitContainer("sidecar", func(c *v1.Container) {
+					c.Image = "local:2.0"
+				})),
+
+				expectedMode: SilentRotation,
+
+				deploymentSpec: buildDeployment(func(depl *api.DeploymentSpec) {
+					depl.Agents.InitContainers = &api.ServerGroupInitContainers{
+						Mode: api.ServerGroupInitContainerIgnoreMode.New(),
+					}
+				}),
+			},
+		}
+
+		runTestCases(t)(testCases...)
+	})
+
+	t.Run("update", func(t *testing.T) {
+		testCases := []TestCase{
+			{
+				name: "Containers with different image but init rotation enforced",
+				spec: buildPodSpec(addInitContainer(api.ServerGroupReservedInitContainerNameUUID, nil), addInitContainer("sidecar", func(c *v1.Container) {
+					c.Image = "local:1.0"
+				})),
+				status: buildPodSpec(addInitContainer(api.ServerGroupReservedInitContainerNameUUID, nil), addInitContainer("sidecar", func(c *v1.Container) {
+					c.Image = "local:2.0"
+				})),
+
+				expectedMode: GracefulRotation,
+
+				deploymentSpec: buildDeployment(func(depl *api.DeploymentSpec) {
+					depl.Agents.InitContainers = &api.ServerGroupInitContainers{
+						Mode: api.ServerGroupInitContainerUpdateMode.New(),
+					}
+				}),
+			},
+			{
+				name: "Core Containers with different image but init rotation enforced",
+				spec: buildPodSpec(addInitContainer(api.ServerGroupReservedInitContainerNameUUID, func(c *v1.Container) {
+					c.Image = "local:1.0"
+				}), addInitContainer("sidecar", func(c *v1.Container) {
+					c.Image = "local:1.0"
+				})),
+				status: buildPodSpec(addInitContainer(api.ServerGroupReservedInitContainerNameUUID, func(c *v1.Container) {
+					c.Image = "local:2.0"
+				}), addInitContainer("sidecar", func(c *v1.Container) {
+					c.Image = "local:1.0"
+				})),
+
+				expectedMode: SilentRotation,
+
+				deploymentSpec: buildDeployment(func(depl *api.DeploymentSpec) {
+					depl.Agents.InitContainers = &api.ServerGroupInitContainers{
+						Mode: api.ServerGroupInitContainerUpdateMode.New(),
+					}
+				}),
+			},
+		}
+
+		runTestCases(t)(testCases...)
+	})
+}

pkg/deployment/rotation/check.go

@@ -34,15 +34,15 @@ import (
 type Mode int
 
 const (
-	EnforcedRotation Mode = iota
-	GracefulRotation
-	InPlaceRotation
+	SkippedRotation Mode = iota
 	SilentRotation
-	SkippedRotation
+	InPlaceRotation
+	GracefulRotation
+	EnforcedRotation
 )
 
 func (m Mode) And(b Mode) Mode {
-	if m < b {
+	if m > b {
 		return m
 	}
 
@@ -64,7 +64,7 @@ func CheckPossible(member api.MemberStatus) bool {
 	return true
 }
 
-func IsRotationRequired(log zerolog.Logger, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, member api.MemberStatus, pod *core.Pod, specTemplate, statusTemplate *api.ArangoMemberPodTemplate) (mode Mode, plan api.Plan, reason string, err error) {
+func IsRotationRequired(log zerolog.Logger, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, member api.MemberStatus, group api.ServerGroup, pod *core.Pod, specTemplate, statusTemplate *api.ArangoMemberPodTemplate) (mode Mode, plan api.Plan, reason string, err error) {
 	// Determine if rotation is required based on plan and actions
 
 	// Set default mode for return value
@@ -123,12 +123,9 @@ func IsRotationRequired(log zerolog.Logger, cachedStatus inspectorInterface.Insp
 		}
 	}
 
-	if statusTemplate.RotationNeeded(specTemplate) {
-		reason = "Pod needs rotation - templates does not match"
-		mode = GracefulRotation
-		log.Info().Str("id", member.ID).Str("Before", member.PodSpecVersion).Msgf(reason)
-		return
+	if mode, plan, err := compare(log, spec, member, group, specTemplate, statusTemplate); err != nil {
+		return SkippedRotation, nil, "", err
+	} else {
+		return mode, plan, "Pod needs rotation", nil
 	}
-
-	return
 }

pkg/deployment/rotation/compare.go

@@ -0,0 +1,101 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package rotation
+
+import (
+	"encoding/json"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
+	"github.com/rs/zerolog"
+	core "k8s.io/api/core/v1"
+)
+
+type compareFuncGen func(deploymentSpec api.DeploymentSpec, group api.ServerGroup, spec, status *core.PodSpec) compareFunc
+type compareFunc func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error)
+
+func generator(deploymentSpec api.DeploymentSpec, group api.ServerGroup, spec, status *core.PodSpec) func(c compareFuncGen) compareFunc {
+	return func(c compareFuncGen) compareFunc {
+		return c(deploymentSpec, group, spec, status)
+	}
+}
+
+func compareFuncs(builder api.ActionBuilder, f ...compareFunc) (mode Mode, plan api.Plan, err error) {
+	for _, q := range f {
+		if m, p, err := q(builder); err != nil {
+			return 0, nil, err
+		} else {
+			mode = mode.And(m)
+			plan = append(plan, p...)
+		}
+	}
+
+	return
+}
+
+func compare(log zerolog.Logger, deploymentSpec api.DeploymentSpec, member api.MemberStatus, group api.ServerGroup, spec, status *api.ArangoMemberPodTemplate) (mode Mode, plan api.Plan, err error) {
+	if spec.Checksum == status.Checksum {
+		return SkippedRotation, nil, nil
+	}
+
+	mode = SkippedRotation
+
+	podStatus := status.PodSpec.DeepCopy()
+
+	// Try to fill fields
+	b := api.NewActionBuilder(group, member.ID)
+
+	g := generator(deploymentSpec, group, &spec.PodSpec.Spec, &podStatus.Spec)
+
+	if m, p, err := compareFuncs(b, g(containersCompare), g(initContainersCompare)); err != nil {
+		log.Err(err).Msg("Error while getting pod diff")
+		return SkippedRotation, nil, err
+	} else {
+		mode = mode.And(m)
+		plan = append(plan, p...)
+	}
+
+	checksum, err := resources.ChecksumArangoPod(deploymentSpec.GetServerGroupSpec(group), resources.CreatePodFromTemplate(podStatus))
+	if err != nil {
+		log.Err(err).Msg("Error while getting pod checksum")
+		return SkippedRotation, nil, err
+	}
+
+	newSpec, err := api.GetArangoMemberPodTemplate(podStatus, checksum)
+	if err != nil {
+		log.Err(err).Msg("Error while getting template")
+		return SkippedRotation, nil, err
+	}
+
+	if spec.RotationNeeded(newSpec) {
+		l := log.Info().Str("id", member.ID).Str("Before", spec.PodSpecChecksum)
+		if d, err := json.Marshal(status); err == nil {
+			l = l.Str("status", string(d))
+		}
+		if d, err := json.Marshal(newSpec); err == nil {
+			l = l.Str("spec", string(d))
+		}
+		l.Msgf("Pod needs rotation - templates does not match")
+		return GracefulRotation, nil, nil
+	}
+
+	return
+}

pkg/deployment/rotation/utils_test.go

@@ -0,0 +1,140 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package rotation
+
+import (
+	"testing"
+
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
+	"github.com/rs/zerolog/log"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/stretchr/testify/require"
+	core "k8s.io/api/core/v1"
+)
+
+type TestCase struct {
+	name         string
+	spec, status *core.PodTemplateSpec
+
+	deploymentSpec api.DeploymentSpec
+	expectedMode   Mode
+	expectedPlan   api.Plan
+	expectedErr    string
+}
+
+func runTestCases(t *testing.T) func(tcs ...TestCase) {
+	return func(tcs ...TestCase) {
+		for _, tc := range tcs {
+			t.Run(tc.name, func(t *testing.T) {
+
+				pspec := newTemplateFromSpec(t, tc.spec, api.ServerGroupAgents, tc.deploymentSpec)
+				pstatus := newTemplateFromSpec(t, tc.status, api.ServerGroupAgents, tc.deploymentSpec)
+
+				mode, plan, err := compare(log.Logger, tc.deploymentSpec, api.MemberStatus{ID: "id"}, api.ServerGroupAgents, pspec, pstatus)
+
+				if tc.expectedErr != "" {
+					require.Error(t, err)
+					require.EqualError(t, err, tc.expectedErr)
+				} else {
+					require.Equal(t, tc.expectedMode, mode)
+
+					switch mode {
+					case InPlaceRotation:
+						require.Len(t, plan, len(tc.expectedPlan))
+
+						for i := range plan {
+							require.Equal(t, tc.expectedPlan[i].Type, plan[i].Type)
+						}
+					}
+				}
+			})
+		}
+	}
+}
+
+func newTemplateFromSpec(t *testing.T, podSpec *core.PodTemplateSpec, group api.ServerGroup, deploymentSpec api.DeploymentSpec) *api.ArangoMemberPodTemplate {
+	checksum, err := resources.ChecksumArangoPod(deploymentSpec.GetServerGroupSpec(group), resources.CreatePodFromTemplate(podSpec))
+	require.NoError(t, err)
+
+	newSpec, err := api.GetArangoMemberPodTemplate(podSpec, checksum)
+	require.NoError(t, err)
+
+	return newSpec
+}
+
+type podSpecBuilder func(pod *core.PodTemplateSpec)
+
+func buildPodSpec(b ...podSpecBuilder) *core.PodTemplateSpec {
+	p := &core.PodTemplateSpec{}
+
+	for _, i := range b {
+		i(p)
+	}
+
+	return p
+}
+
+func addContainer(name string, f func(c *core.Container)) podSpecBuilder {
+	return func(pod *core.PodTemplateSpec) {
+		var c core.Container
+
+		c.Name = name
+
+		if f != nil {
+			f(&c)
+		}
+
+		pod.Spec.Containers = append(pod.Spec.Containers, c)
+	}
+}
+
+func addInitContainer(name string, f func(c *core.Container)) podSpecBuilder {
+	return func(pod *core.PodTemplateSpec) {
+		var c core.Container
+
+		c.Name = name
+
+		if f != nil {
+			f(&c)
+		}
+
+		pod.Spec.InitContainers = append(pod.Spec.InitContainers, c)
+	}
+}
+
+func addSidecarWithImage(name, image string) podSpecBuilder {
+	return addContainer(name, func(c *core.Container) {
+		c.Image = image
+	})
+}
+
+type deploymentBuilder func(depl *api.DeploymentSpec)
+
+func buildDeployment(b ...deploymentBuilder) api.DeploymentSpec {
+	p := api.DeploymentSpec{}
+
+	for _, i := range b {
+		i(&p)
+	}
+
+	return p
+}

pkg/util/checksum.go

@@ -25,6 +25,8 @@ package util
 import (
 	"crypto/sha256"
 	"fmt"
+
+	"k8s.io/apimachinery/pkg/util/json"
 )
 
 func SHA256FromString(data string) string {
@@ -34,3 +36,25 @@ func SHA256FromString(data string) string {
 func SHA256(data []byte) string {
 	return fmt.Sprintf("%0x", sha256.Sum256(data))
 }
+
+func SHA256FromJSON(a interface{}) (string, error) {
+	d, err := json.Marshal(a)
+	if err != nil {
+		return "", err
+	}
+
+	return SHA256(d), nil
+}
+
+func CompareJSON(a, b interface{}) (bool, error) {
+	ad, err := SHA256FromJSON(a)
+	if err != nil {
+		return false, err
+	}
+	bd, err := SHA256FromJSON(b)
+	if err != nil {
+		return false, err
+	}
+
+	return ad == bd, nil
+}

pkg/util/k8sutil/pods.go

@@ -25,11 +25,13 @@ package k8sutil
 
 import (
 	"context"
-	"crypto/sha256"
 	"fmt"
 	"path/filepath"
+	"strings"
 	"time"
 
+	"github.com/arangodb/kube-arangodb/pkg/util"
+
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/interfaces"
@@ -65,6 +67,9 @@ const (
 	ClusterJWTSecretVolumeMountDir  = "/secrets/cluster/jwt"
 	ExporterJWTVolumeMountDir       = "/secrets/exporter/jwt"
 	MasterJWTSecretVolumeMountDir   = "/secrets/master/jwt"
+
+	ServerContainerConditionContainersNotReady = "ContainersNotReady"
+	ServerContainerConditionPrefix             = "containers with unready status: "
 )
 
 // IsPodReady returns true if the PodReady condition on
@@ -74,6 +79,35 @@ func IsPodReady(pod *core.Pod) bool {
 	return condition != nil && condition.Status == core.ConditionTrue
 }
 
+// IsContainerReady returns true if the PodReady condition on
+// the given pod is set to true.
+func IsContainerReady(pod *core.Pod, container string) bool {
+	condition := getPodCondition(&pod.Status, core.PodReady)
+	if condition == nil {
+		return false
+	}
+
+	if condition.Status == core.ConditionTrue {
+		return true
+	}
+
+	if !IsContainerRunning(pod, container) {
+		return false
+	}
+
+	switch condition.Reason {
+	case ServerContainerConditionContainersNotReady:
+		if strings.HasPrefix(condition.Message, ServerContainerConditionPrefix) {
+			n := strings.TrimPrefix(condition.Message, ServerContainerConditionPrefix)
+
+			return !strings.Contains(n, container)
+		}
+		return false
+	default:
+		return false
+	}
+}
+
 // GetPodByName returns pod if it exists among the pods' list
 // Returns false if not found.
 func GetPodByName(pods []core.Pod, podName string) (core.Pod, bool) {
@@ -87,8 +121,13 @@ func GetPodByName(pods []core.Pod, podName string) (core.Pod, bool) {
 
 // IsPodServerContainerRunning returns true if the arangodb container of the pod is still running
 func IsPodServerContainerRunning(pod *core.Pod) bool {
+	return IsContainerRunning(pod, ServerContainerName)
+}
+
+// IsContainerRunning returns true if the container of the pod is still running
+func IsContainerRunning(pod *core.Pod, name string) bool {
 	for _, c := range pod.Status.ContainerStatuses {
-		if c.Name != ServerContainerName {
+		if c.Name != name {
 			continue
 		}
 
@@ -420,7 +459,7 @@ func GetPodSpecChecksum(podSpec core.PodSpec) (string, error) {
 		return "", err
 	}
 
-	return fmt.Sprintf("%0x", sha256.Sum256(data)), nil
+	return util.SHA256(data), nil
 }
 
 // CreatePod adds an owner to the given pod and calls the k8s api-server to created it.