[Feature] Allow to disable external port (#906)

CHANGELOG.md

@@ -8,6 +8,7 @@
 - (ARM64) Add support for ARM64 enablement
 - (Cleanup) Reorganize main reconciliation context
 - (Bugfix) Unreachable condition
+- (Feature) Allow to disable external port (sidecar managed connection)
 
 ## [1.2.7](https://github.com/arangodb/kube-arangodb/tree/1.2.7) (2022-01-17)
 - Add Plan BackOff functionality

pkg/apis/deployment/v1/conditions.go

@@ -22,7 +22,7 @@ package v1
 
 import (
 	"github.com/arangodb/kube-arangodb/pkg/util"
-	v1 "k8s.io/api/core/v1"
+	core "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -102,7 +102,7 @@ type Condition struct {
 	// Type of  condition.
 	Type ConditionType `json:"type"`
 	// Status of the condition, one of True, False, Unknown.
-	Status v1.ConditionStatus `json:"status"`
+	Status core.ConditionStatus `json:"status"`
 	// The last time this condition was updated.
 	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
 	// Last time the condition transitioned from one status to another.
@@ -116,7 +116,18 @@ type Condition struct {
 }
 
 func (c Condition) IsTrue() bool {
-	return c.Status == v1.ConditionTrue
+	return c.Status == core.ConditionTrue
+}
+
+// Equal checks for equality
+func (c Condition) Equal(other Condition) bool {
+	return c.Type == other.Type &&
+		c.Status == other.Status &&
+		util.TimeCompareEqual(c.LastUpdateTime, other.LastUpdateTime) &&
+		util.TimeCompareEqual(c.LastTransitionTime, other.LastTransitionTime) &&
+		c.Reason == other.Reason &&
+		c.Message == other.Message &&
+		c.Hash == other.Hash
 }
 
 // ConditionList is a list of conditions.
@@ -143,30 +154,20 @@ func (list ConditionList) Equal(other ConditionList) bool {
 	return true
 }
 
-// Equal checks for equality
-func (c Condition) Equal(other Condition) bool {
-	return c.Type == other.Type &&
-		c.Status == other.Status &&
-		util.TimeCompareEqual(c.LastUpdateTime, other.LastUpdateTime) &&
-		util.TimeCompareEqual(c.LastTransitionTime, other.LastTransitionTime) &&
-		c.Reason == other.Reason &&
-		c.Message == other.Message &&
-		c.Hash == other.Hash
-}
-
 // IsTrue return true when a condition with given type exists and its status is `True`.
 func (list ConditionList) IsTrue(conditionType ConditionType) bool {
 	c, found := list.Get(conditionType)
 	return found && c.IsTrue()
 }
 
-// GetValue returns *bool value in case if condition exists, nil otherwise
-func (list ConditionList) GetValue(conditionType ConditionType) *bool {
-	c, found := list.Get(conditionType)
-	if found {
-		return util.NewBool(c.IsTrue())
+// Check create a condition checker.
+func (list ConditionList) Check(conditionType ConditionType) ConditionCheck {
+	c, ok := list.Get(conditionType)
+
+	return conditionCheck{
+		condition: c,
+		exists:    ok,
 	}
-	return nil
 }
 
 // Get a condition by type.
@@ -211,9 +212,9 @@ func (list ConditionList) Index(conditionType ConditionType) int {
 
 func (list *ConditionList) update(conditionType ConditionType, status bool, reason, message, hash string) bool {
 	src := *list
-	statusX := v1.ConditionFalse
+	statusX := core.ConditionFalse
 	if status {
-		statusX = v1.ConditionTrue
+		statusX = core.ConditionTrue
 	}
 
 	index := list.Index(conditionType)

pkg/apis/deployment/v1/conditions_check.go

@@ -0,0 +1,102 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+import "time"
+
+type ConditionCheck interface {
+	Evaluate() bool
+
+	Exists() ConditionCheck
+	IsTrue() ConditionCheck
+	IsFalse() ConditionCheck
+
+	LastTransition(d time.Duration) ConditionCheck
+}
+
+var _ ConditionCheck = conditionCheck{}
+
+type conditionCheck struct {
+	condition Condition
+	exists    bool
+}
+
+func (c conditionCheck) LastTransition(d time.Duration) ConditionCheck {
+	if c.exists && (!c.condition.LastTransitionTime.IsZero() && time.Since(c.condition.LastTransitionTime.Time) >= d) {
+		return c
+	}
+
+	return newConditionCheckConst(false)
+}
+
+func (c conditionCheck) IsTrue() ConditionCheck {
+	if c.condition.IsTrue() {
+		return c
+	}
+
+	return newConditionCheckConst(false)
+}
+
+func (c conditionCheck) IsFalse() ConditionCheck {
+	if !c.condition.IsTrue() {
+		return c
+	}
+
+	return newConditionCheckConst(false)
+}
+
+func (c conditionCheck) Evaluate() bool {
+	return true
+}
+
+func (c conditionCheck) Exists() ConditionCheck {
+	if c.exists {
+		return c
+	}
+
+	return newConditionCheckConst(false)
+}
+
+func newConditionCheckConst(c bool) ConditionCheck {
+	return conditionCheckConst(c)
+}
+
+type conditionCheckConst bool
+
+func (c conditionCheckConst) LastTransition(d time.Duration) ConditionCheck {
+	return c
+}
+
+func (c conditionCheckConst) IsTrue() ConditionCheck {
+	return c
+}
+
+func (c conditionCheckConst) IsFalse() ConditionCheck {
+	return c
+}
+
+func (c conditionCheckConst) Evaluate() bool {
+	return bool(c)
+}
+
+func (c conditionCheckConst) Exists() ConditionCheck {
+	return c
+}

pkg/apis/deployment/v1/conditions_check_test.go

@@ -0,0 +1,216 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	core "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func Test_ConditionCheck(t *testing.T) {
+	type testCase struct {
+		name string
+
+		conditions ConditionList
+
+		check func(l ConditionList) ConditionCheck
+
+		expected bool
+	}
+
+	var ct ConditionType = "test"
+
+	cases := []testCase{
+		{
+			name: "IsTrue when true & exists",
+			conditions: ConditionList{
+				{
+					Type:   ct,
+					Status: core.ConditionTrue,
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).IsTrue()
+			},
+			expected: true,
+		},
+		{
+			name: "IsFalse when false & exists",
+			conditions: ConditionList{
+				{
+					Type:   ct,
+					Status: core.ConditionFalse,
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).IsFalse()
+			},
+			expected: true,
+		},
+		{
+			name: "IsTrue when does not exists",
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).IsTrue()
+			},
+			expected: false,
+		},
+		{
+			name: "IsFalse when does not exists",
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).IsFalse()
+			},
+			expected: true,
+		},
+		{
+			name: "explicit IsTrue when true & exists",
+			conditions: ConditionList{
+				{
+					Type:   ct,
+					Status: core.ConditionTrue,
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).Exists().IsTrue()
+			},
+			expected: true,
+		},
+		{
+			name: "explicit IsFalse when false & exists",
+			conditions: ConditionList{
+				{
+					Type:   ct,
+					Status: core.ConditionFalse,
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).Exists().IsFalse()
+			},
+			expected: true,
+		},
+		{
+			name: "explicit IsTrue when does not exists",
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).Exists().IsTrue()
+			},
+			expected: false,
+		},
+		{
+			name: "explicit IsFalse when does not exists",
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).Exists().IsFalse()
+			},
+			expected: false,
+		},
+		{
+			name: "transitionTime - with current time and duration set to 0",
+			conditions: ConditionList{
+				{
+					Type:               ct,
+					LastTransitionTime: meta.NewTime(time.Now()),
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).LastTransition(0)
+			},
+			expected: true,
+		},
+		{
+			name: "transitionTime - with current time and duration set to 1s",
+			conditions: ConditionList{
+				{
+					Type:               ct,
+					LastTransitionTime: meta.NewTime(time.Now()),
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).LastTransition(time.Second)
+			},
+			expected: false,
+		},
+		{
+			name: "transitionTime - with current time and duration set to 1s",
+			conditions: ConditionList{
+				{
+					Type:               ct,
+					LastTransitionTime: meta.NewTime(time.Now()),
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).LastTransition(time.Second)
+			},
+			expected: false,
+		},
+		{
+			name: "transitionTime - with zero time",
+			conditions: ConditionList{
+				{
+					Type: ct,
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).LastTransition(0)
+			},
+			expected: false,
+		},
+		{
+			name: "transitionTime - with current time and duration set to 15s",
+			conditions: ConditionList{
+				{
+					Type:               ct,
+					LastTransitionTime: meta.NewTime(time.Now().Add(-15 * time.Second)),
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).LastTransition(15 * time.Second)
+			},
+			expected: true,
+		},
+		{
+			name: "transitionTime - with current time and duration set to 14.75s",
+			conditions: ConditionList{
+				{
+					Type:               ct,
+					LastTransitionTime: meta.NewTime(time.Now().Add(-15*time.Second + 250*time.Millisecond)),
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).LastTransition(15 * time.Second)
+			},
+			expected: false,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			r := c.check(c.conditions).Evaluate()
+
+			if c.expected {
+				require.True(t, r)
+			} else {
+				require.False(t, r)
+			}
+		})
+	}
+}

pkg/apis/deployment/v1/plan.go

@@ -24,7 +24,9 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util"
 	"github.com/dchest/uniuri"
 	"k8s.io/apimachinery/pkg/api/equality"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/uuid"
 )
 
 // ActionPriority define action priority
@@ -197,10 +199,16 @@ const (
 	MemberIDPreviousAction = "@previous"
 )
 
+const (
+	ParamPodUID = "PodUID"
+)
+
 // Action represents a single action to be taken to update a deployment.
 type Action struct {
 	// ID of this action (unique for every action)
 	ID string `json:"id"`
+	// SetID define the unique ID of current action set
+	SetID types.UID `json:"setID,omitempty"`
 	// Type of action.
 	Type ActionType `json:"type"`
 	// ID reference of the member involved in this action (if any)
@@ -208,9 +216,9 @@ type Action struct {
 	// Group involved in this action
 	Group ServerGroup `json:"group,omitempty"`
 	// CreationTime is set the when the action is created.
-	CreationTime metav1.Time `json:"creationTime"`
+	CreationTime meta.Time `json:"creationTime"`
 	// StartTime is set the when the action has been started, but needs to wait to be finished.
-	StartTime *metav1.Time `json:"startTime,omitempty"`
+	StartTime *meta.Time `json:"startTime,omitempty"`
 	// Reason for this action
 	Reason string `json:"reason,omitempty"`
 	// Image used in can of a SetCurrentImage action.
@@ -254,6 +262,15 @@ func (a Action) GetParam(key string) (string, bool) {
 	return i, ok
 }
 
+// NewActionSet add new SetID vale to the actions
+func NewActionSet(actions ...Action) []Action {
+	sid := uuid.NewUUID()
+	for id := range actions {
+		actions[id].SetID = sid
+	}
+	return actions
+}
+
 // NewAction instantiates a new Action.
 func NewAction(actionType ActionType, group ServerGroup, memberID string, reason ...string) Action {
 	a := Action{
@@ -261,7 +278,7 @@ func NewAction(actionType ActionType, group ServerGroup, memberID string, reason
 		Type:         actionType,
 		MemberID:     memberID,
 		Group:        group,
-		CreationTime: metav1.Now(),
+		CreationTime: meta.Now(),
 	}
 	if len(reason) != 0 {
 		a.Reason = reason[0]

pkg/apis/deployment/v1/plan_test.go

@@ -0,0 +1,36 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_Action_Marshal(t *testing.T) {
+	var a Action
+
+	data, err := json.Marshal(a)
+	require.NoError(t, err)
+	require.Equal(t, `{"id":"","type":"","creationTime":null}`, string(data))
+}

pkg/apis/deployment/v1/server_group_spec.go

@@ -145,8 +145,12 @@ type ServerGroupSpec struct {
 	ShutdownMethod *ServerGroupShutdownMethod `json:"shutdownMethod,omitempty"`
 	// ShutdownDelay define how long operator should delay finalizer removal after shutdown
 	ShutdownDelay *int `json:"shutdownDelay,omitempty"`
-	// InternalPort define port used in internal communication, can be accessed over localhost via sidecar
+	// InternalPort define port used in internal communication, can be accessed over localhost via sidecar. Only for ArangoD members
 	InternalPort *int `json:"internalPort,omitempty"`
+	// InternalPortProtocol define protocol of port used in internal communication, can be accessed over localhost via sidecar. Only for ArangoD members
+	InternalPortProtocol *ServerGroupPortProtocol `json:"internalPortProtocol,omitempty"`
+	// ExternalPortEnabled if external port should be enabled. If is set to false, ports needs to be exposed via sidecar. Only for ArangoD members
+	ExternalPortEnabled *bool `json:"externalPortEnabled,omitempty"`
 	// AllowMemberRecreation allows to recreate member. Value is used only for Coordinator and DBServer with default to True, for all other groups set to false.
 	AllowMemberRecreation *bool `json:"allowMemberRecreation,omitempty"`
 	// TerminationGracePeriodSeconds override default TerminationGracePeriodSeconds for pods - via silent rotation
@@ -517,6 +521,9 @@ func (s ServerGroupSpec) Validate(group ServerGroup, used bool, mode DeploymentM
 		return errors.WithStack(errors.Wrapf(ValidationError, "Invalid count value %d for un-used group. Expected 0", s.GetCount()))
 	}
 	if port := s.InternalPort; port != nil {
+		if err := s.InternalPortProtocol.Validate(); err != nil {
+			return errors.Wrapf(err, "Validation of InternalPortProtocol failed")
+		}
 		switch p := *port; p {
 		case 8529:
 			return errors.WithStack(errors.Wrapf(ValidationError, "Port %d already in use", p))
@@ -712,3 +719,12 @@ func (s ServerGroupSpec) GetTerminationGracePeriod(group ServerGroup) time.Durat
 		return time.Second * time.Duration(*v)
 	}
 }
+
+// GetExternalPortEnabled returns value of ExternalPortEnabled. If ExternalPortEnabled is nil true is returned
+func (s ServerGroupSpec) GetExternalPortEnabled() bool {
+	if v := s.ExternalPortEnabled; v == nil {
+		return true
+	} else {
+		return *v
+	}
+}

pkg/apis/deployment/v1/server_group_spec_port_proto.go

@@ -0,0 +1,62 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+import "github.com/pkg/errors"
+
+// ServerGroupPortProtocol define supported protocols of listeners
+type ServerGroupPortProtocol string
+
+// Get returns current protocol. If is nil then default is returned
+func (s *ServerGroupPortProtocol) Get() ServerGroupPortProtocol {
+	if s == nil {
+		return ServerGroupPortProtocolDefault
+	}
+	return *s
+}
+
+// New returns pointer to copy of protocol value
+func (s ServerGroupPortProtocol) New() *ServerGroupPortProtocol {
+	return &s
+}
+
+// Validate validates if protocol is known and have valid value
+func (s *ServerGroupPortProtocol) Validate() error {
+	if s == nil {
+		return nil
+	}
+
+	switch v := *s; v {
+	case ServerGroupPortProtocolHTTP, ServerGroupPortProtocolHTTPS:
+		return nil
+	default:
+		return errors.Errorf("Unknown proto %s", v)
+	}
+}
+
+const (
+	// ServerGroupPortProtocolHTTP defines HTTP protocol
+	ServerGroupPortProtocolHTTP ServerGroupPortProtocol = "http"
+	// ServerGroupPortProtocolHTTPS defines HTTPS protocol
+	ServerGroupPortProtocolHTTPS ServerGroupPortProtocol = "https"
+	// ServerGroupPortProtocolDefault defines default (HTTP) protocol
+	ServerGroupPortProtocolDefault = ServerGroupPortProtocolHTTP
+)

pkg/apis/deployment/v1/zz_generated.deepcopy.go

@@ -2063,6 +2063,16 @@ func (in *ServerGroupSpec) DeepCopyInto(out *ServerGroupSpec) {
 		*out = new(int)
 		**out = **in
 	}
+	if in.InternalPortProtocol != nil {
+		in, out := &in.InternalPortProtocol, &out.InternalPortProtocol
+		*out = new(ServerGroupPortProtocol)
+		**out = **in
+	}
+	if in.ExternalPortEnabled != nil {
+		in, out := &in.ExternalPortEnabled, &out.ExternalPortEnabled
+		*out = new(bool)
+		**out = **in
+	}
 	if in.AllowMemberRecreation != nil {
 		in, out := &in.AllowMemberRecreation, &out.AllowMemberRecreation
 		*out = new(bool)

pkg/apis/deployment/v2alpha1/conditions.go

@@ -22,7 +22,7 @@ package v2alpha1
 
 import (
 	"github.com/arangodb/kube-arangodb/pkg/util"
-	v1 "k8s.io/api/core/v1"
+	core "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -102,7 +102,7 @@ type Condition struct {
 	// Type of  condition.
 	Type ConditionType `json:"type"`
 	// Status of the condition, one of True, False, Unknown.
-	Status v1.ConditionStatus `json:"status"`
+	Status core.ConditionStatus `json:"status"`
 	// The last time this condition was updated.
 	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
 	// Last time the condition transitioned from one status to another.
@@ -116,7 +116,18 @@ type Condition struct {
 }
 
 func (c Condition) IsTrue() bool {
-	return c.Status == v1.ConditionTrue
+	return c.Status == core.ConditionTrue
+}
+
+// Equal checks for equality
+func (c Condition) Equal(other Condition) bool {
+	return c.Type == other.Type &&
+		c.Status == other.Status &&
+		util.TimeCompareEqual(c.LastUpdateTime, other.LastUpdateTime) &&
+		util.TimeCompareEqual(c.LastTransitionTime, other.LastTransitionTime) &&
+		c.Reason == other.Reason &&
+		c.Message == other.Message &&
+		c.Hash == other.Hash
 }
 
 // ConditionList is a list of conditions.
@@ -143,30 +154,20 @@ func (list ConditionList) Equal(other ConditionList) bool {
 	return true
 }
 
-// Equal checks for equality
-func (c Condition) Equal(other Condition) bool {
-	return c.Type == other.Type &&
-		c.Status == other.Status &&
-		util.TimeCompareEqual(c.LastUpdateTime, other.LastUpdateTime) &&
-		util.TimeCompareEqual(c.LastTransitionTime, other.LastTransitionTime) &&
-		c.Reason == other.Reason &&
-		c.Message == other.Message &&
-		c.Hash == other.Hash
-}
-
 // IsTrue return true when a condition with given type exists and its status is `True`.
 func (list ConditionList) IsTrue(conditionType ConditionType) bool {
 	c, found := list.Get(conditionType)
 	return found && c.IsTrue()
 }
 
-// GetValue returns *bool value in case if condition exists, nil otherwise
-func (list ConditionList) GetValue(conditionType ConditionType) *bool {
-	c, found := list.Get(conditionType)
-	if found {
-		return util.NewBool(c.IsTrue())
+// Check create a condition checker.
+func (list ConditionList) Check(conditionType ConditionType) ConditionCheck {
+	c, ok := list.Get(conditionType)
+
+	return conditionCheck{
+		condition: c,
+		exists:    ok,
 	}
-	return nil
 }
 
 // Get a condition by type.
@@ -211,9 +212,9 @@ func (list ConditionList) Index(conditionType ConditionType) int {
 
 func (list *ConditionList) update(conditionType ConditionType, status bool, reason, message, hash string) bool {
 	src := *list
-	statusX := v1.ConditionFalse
+	statusX := core.ConditionFalse
 	if status {
-		statusX = v1.ConditionTrue
+		statusX = core.ConditionTrue
 	}
 
 	index := list.Index(conditionType)

pkg/apis/deployment/v2alpha1/conditions_check.go

@@ -0,0 +1,102 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v2alpha1
+
+import "time"
+
+type ConditionCheck interface {
+	Evaluate() bool
+
+	Exists() ConditionCheck
+	IsTrue() ConditionCheck
+	IsFalse() ConditionCheck
+
+	LastTransition(d time.Duration) ConditionCheck
+}
+
+var _ ConditionCheck = conditionCheck{}
+
+type conditionCheck struct {
+	condition Condition
+	exists    bool
+}
+
+func (c conditionCheck) LastTransition(d time.Duration) ConditionCheck {
+	if c.exists && (!c.condition.LastTransitionTime.IsZero() && time.Since(c.condition.LastTransitionTime.Time) >= d) {
+		return c
+	}
+
+	return newConditionCheckConst(false)
+}
+
+func (c conditionCheck) IsTrue() ConditionCheck {
+	if c.condition.IsTrue() {
+		return c
+	}
+
+	return newConditionCheckConst(false)
+}
+
+func (c conditionCheck) IsFalse() ConditionCheck {
+	if !c.condition.IsTrue() {
+		return c
+	}
+
+	return newConditionCheckConst(false)
+}
+
+func (c conditionCheck) Evaluate() bool {
+	return true
+}
+
+func (c conditionCheck) Exists() ConditionCheck {
+	if c.exists {
+		return c
+	}
+
+	return newConditionCheckConst(false)
+}
+
+func newConditionCheckConst(c bool) ConditionCheck {
+	return conditionCheckConst(c)
+}
+
+type conditionCheckConst bool
+
+func (c conditionCheckConst) LastTransition(d time.Duration) ConditionCheck {
+	return c
+}
+
+func (c conditionCheckConst) IsTrue() ConditionCheck {
+	return c
+}
+
+func (c conditionCheckConst) IsFalse() ConditionCheck {
+	return c
+}
+
+func (c conditionCheckConst) Evaluate() bool {
+	return bool(c)
+}
+
+func (c conditionCheckConst) Exists() ConditionCheck {
+	return c
+}

pkg/apis/deployment/v2alpha1/conditions_check_test.go

@@ -0,0 +1,216 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v2alpha1
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	core "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func Test_ConditionCheck(t *testing.T) {
+	type testCase struct {
+		name string
+
+		conditions ConditionList
+
+		check func(l ConditionList) ConditionCheck
+
+		expected bool
+	}
+
+	var ct ConditionType = "test"
+
+	cases := []testCase{
+		{
+			name: "IsTrue when true & exists",
+			conditions: ConditionList{
+				{
+					Type:   ct,
+					Status: core.ConditionTrue,
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).IsTrue()
+			},
+			expected: true,
+		},
+		{
+			name: "IsFalse when false & exists",
+			conditions: ConditionList{
+				{
+					Type:   ct,
+					Status: core.ConditionFalse,
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).IsFalse()
+			},
+			expected: true,
+		},
+		{
+			name: "IsTrue when does not exists",
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).IsTrue()
+			},
+			expected: false,
+		},
+		{
+			name: "IsFalse when does not exists",
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).IsFalse()
+			},
+			expected: true,
+		},
+		{
+			name: "explicit IsTrue when true & exists",
+			conditions: ConditionList{
+				{
+					Type:   ct,
+					Status: core.ConditionTrue,
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).Exists().IsTrue()
+			},
+			expected: true,
+		},
+		{
+			name: "explicit IsFalse when false & exists",
+			conditions: ConditionList{
+				{
+					Type:   ct,
+					Status: core.ConditionFalse,
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).Exists().IsFalse()
+			},
+			expected: true,
+		},
+		{
+			name: "explicit IsTrue when does not exists",
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).Exists().IsTrue()
+			},
+			expected: false,
+		},
+		{
+			name: "explicit IsFalse when does not exists",
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).Exists().IsFalse()
+			},
+			expected: false,
+		},
+		{
+			name: "transitionTime - with current time and duration set to 0",
+			conditions: ConditionList{
+				{
+					Type:               ct,
+					LastTransitionTime: meta.NewTime(time.Now()),
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).LastTransition(0)
+			},
+			expected: true,
+		},
+		{
+			name: "transitionTime - with current time and duration set to 1s",
+			conditions: ConditionList{
+				{
+					Type:               ct,
+					LastTransitionTime: meta.NewTime(time.Now()),
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).LastTransition(time.Second)
+			},
+			expected: false,
+		},
+		{
+			name: "transitionTime - with current time and duration set to 1s",
+			conditions: ConditionList{
+				{
+					Type:               ct,
+					LastTransitionTime: meta.NewTime(time.Now()),
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).LastTransition(time.Second)
+			},
+			expected: false,
+		},
+		{
+			name: "transitionTime - with zero time",
+			conditions: ConditionList{
+				{
+					Type: ct,
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).LastTransition(0)
+			},
+			expected: false,
+		},
+		{
+			name: "transitionTime - with current time and duration set to 15s",
+			conditions: ConditionList{
+				{
+					Type:               ct,
+					LastTransitionTime: meta.NewTime(time.Now().Add(-15 * time.Second)),
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).LastTransition(15 * time.Second)
+			},
+			expected: true,
+		},
+		{
+			name: "transitionTime - with current time and duration set to 14.75s",
+			conditions: ConditionList{
+				{
+					Type:               ct,
+					LastTransitionTime: meta.NewTime(time.Now().Add(-15*time.Second + 250*time.Millisecond)),
+				},
+			},
+			check: func(l ConditionList) ConditionCheck {
+				return l.Check(ct).LastTransition(15 * time.Second)
+			},
+			expected: false,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			r := c.check(c.conditions).Evaluate()
+
+			if c.expected {
+				require.True(t, r)
+			} else {
+				require.False(t, r)
+			}
+		})
+	}
+}

pkg/apis/deployment/v2alpha1/plan.go

@@ -24,7 +24,9 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util"
 	"github.com/dchest/uniuri"
 	"k8s.io/apimachinery/pkg/api/equality"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/uuid"
 )
 
 // ActionPriority define action priority
@@ -197,10 +199,16 @@ const (
 	MemberIDPreviousAction = "@previous"
 )
 
+const (
+	ParamPodUID = "PodUID"
+)
+
 // Action represents a single action to be taken to update a deployment.
 type Action struct {
 	// ID of this action (unique for every action)
 	ID string `json:"id"`
+	// SetID define the unique ID of current action set
+	SetID types.UID `json:"setID,omitempty"`
 	// Type of action.
 	Type ActionType `json:"type"`
 	// ID reference of the member involved in this action (if any)
@@ -208,9 +216,9 @@ type Action struct {
 	// Group involved in this action
 	Group ServerGroup `json:"group,omitempty"`
 	// CreationTime is set the when the action is created.
-	CreationTime metav1.Time `json:"creationTime"`
+	CreationTime meta.Time `json:"creationTime"`
 	// StartTime is set the when the action has been started, but needs to wait to be finished.
-	StartTime *metav1.Time `json:"startTime,omitempty"`
+	StartTime *meta.Time `json:"startTime,omitempty"`
 	// Reason for this action
 	Reason string `json:"reason,omitempty"`
 	// Image used in can of a SetCurrentImage action.
@@ -254,6 +262,15 @@ func (a Action) GetParam(key string) (string, bool) {
 	return i, ok
 }
 
+// NewActionSet add new SetID vale to the actions
+func NewActionSet(actions ...Action) []Action {
+	sid := uuid.NewUUID()
+	for id := range actions {
+		actions[id].SetID = sid
+	}
+	return actions
+}
+
 // NewAction instantiates a new Action.
 func NewAction(actionType ActionType, group ServerGroup, memberID string, reason ...string) Action {
 	a := Action{
@@ -261,7 +278,7 @@ func NewAction(actionType ActionType, group ServerGroup, memberID string, reason
 		Type:         actionType,
 		MemberID:     memberID,
 		Group:        group,
-		CreationTime: metav1.Now(),
+		CreationTime: meta.Now(),
 	}
 	if len(reason) != 0 {
 		a.Reason = reason[0]

pkg/apis/deployment/v2alpha1/plan_test.go

@@ -0,0 +1,36 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v2alpha1
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_Action_Marshal(t *testing.T) {
+	var a Action
+
+	data, err := json.Marshal(a)
+	require.NoError(t, err)
+	require.Equal(t, `{"id":"","type":"","creationTime":null}`, string(data))
+}

pkg/apis/deployment/v2alpha1/server_group_spec.go

@@ -145,8 +145,12 @@ type ServerGroupSpec struct {
 	ShutdownMethod *ServerGroupShutdownMethod `json:"shutdownMethod,omitempty"`
 	// ShutdownDelay define how long operator should delay finalizer removal after shutdown
 	ShutdownDelay *int `json:"shutdownDelay,omitempty"`
-	// InternalPort define port used in internal communication, can be accessed over localhost via sidecar
+	// InternalPort define port used in internal communication, can be accessed over localhost via sidecar. Only for ArangoD members
 	InternalPort *int `json:"internalPort,omitempty"`
+	// InternalPortProtocol define protocol of port used in internal communication, can be accessed over localhost via sidecar. Only for ArangoD members
+	InternalPortProtocol *ServerGroupPortProtocol `json:"internalPortProtocol,omitempty"`
+	// ExternalPortEnabled if external port should be enabled. If is set to false, ports needs to be exposed via sidecar. Only for ArangoD members
+	ExternalPortEnabled *bool `json:"externalPortEnabled,omitempty"`
 	// AllowMemberRecreation allows to recreate member. Value is used only for Coordinator and DBServer with default to True, for all other groups set to false.
 	AllowMemberRecreation *bool `json:"allowMemberRecreation,omitempty"`
 	// TerminationGracePeriodSeconds override default TerminationGracePeriodSeconds for pods - via silent rotation
@@ -517,6 +521,9 @@ func (s ServerGroupSpec) Validate(group ServerGroup, used bool, mode DeploymentM
 		return errors.WithStack(errors.Wrapf(ValidationError, "Invalid count value %d for un-used group. Expected 0", s.GetCount()))
 	}
 	if port := s.InternalPort; port != nil {
+		if err := s.InternalPortProtocol.Validate(); err != nil {
+			return errors.Wrapf(err, "Validation of InternalPortProtocol failed")
+		}
 		switch p := *port; p {
 		case 8529:
 			return errors.WithStack(errors.Wrapf(ValidationError, "Port %d already in use", p))
@@ -712,3 +719,12 @@ func (s ServerGroupSpec) GetTerminationGracePeriod(group ServerGroup) time.Durat
 		return time.Second * time.Duration(*v)
 	}
 }
+
+// GetExternalPortEnabled returns value of ExternalPortEnabled. If ExternalPortEnabled is nil true is returned
+func (s ServerGroupSpec) GetExternalPortEnabled() bool {
+	if v := s.ExternalPortEnabled; v == nil {
+		return true
+	} else {
+		return *v
+	}
+}

pkg/apis/deployment/v2alpha1/server_group_spec_port_proto.go

@@ -0,0 +1,62 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v2alpha1
+
+import "github.com/pkg/errors"
+
+// ServerGroupPortProtocol define supported protocols of listeners
+type ServerGroupPortProtocol string
+
+// Get returns current protocol. If is nil then default is returned
+func (s *ServerGroupPortProtocol) Get() ServerGroupPortProtocol {
+	if s == nil {
+		return ServerGroupPortProtocolDefault
+	}
+	return *s
+}
+
+// New returns pointer to copy of protocol value
+func (s ServerGroupPortProtocol) New() *ServerGroupPortProtocol {
+	return &s
+}
+
+// Validate validates if protocol is known and have valid value
+func (s *ServerGroupPortProtocol) Validate() error {
+	if s == nil {
+		return nil
+	}
+
+	switch v := *s; v {
+	case ServerGroupPortProtocolHTTP, ServerGroupPortProtocolHTTPS:
+		return nil
+	default:
+		return errors.Errorf("Unknown proto %s", v)
+	}
+}
+
+const (
+	// ServerGroupPortProtocolHTTP defines HTTP protocol
+	ServerGroupPortProtocolHTTP ServerGroupPortProtocol = "http"
+	// ServerGroupPortProtocolHTTPS defines HTTPS protocol
+	ServerGroupPortProtocolHTTPS ServerGroupPortProtocol = "https"
+	// ServerGroupPortProtocolDefault defines default (HTTP) protocol
+	ServerGroupPortProtocolDefault = ServerGroupPortProtocolHTTP
+)

pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go

@@ -2063,6 +2063,16 @@ func (in *ServerGroupSpec) DeepCopyInto(out *ServerGroupSpec) {
 		*out = new(int)
 		**out = **in
 	}
+	if in.InternalPortProtocol != nil {
+		in, out := &in.InternalPortProtocol, &out.InternalPortProtocol
+		*out = new(ServerGroupPortProtocol)
+		**out = **in
+	}
+	if in.ExternalPortEnabled != nil {
+		in, out := &in.ExternalPortEnabled, &out.ExternalPortEnabled
+		*out = new(bool)
+		**out = **in
+	}
 	if in.AllowMemberRecreation != nil {
 		in, out := &in.AllowMemberRecreation, &out.AllowMemberRecreation
 		*out = new(bool)

pkg/deployment/deployment_inspector.go

@@ -116,6 +116,7 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval
 		d.apiObject = updated
 
 		d.RefreshState(ctxReconciliation, updated.Status.Members.AsList())
+		d.Log(d.deps.Log)
 
 		inspectNextInterval, err := d.inspectDeploymentWithError(ctxReconciliation, nextInterval, cachedStatus)
 		if err != nil {

pkg/deployment/member/state.go

@@ -27,11 +27,14 @@ import (
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/reconciler"
 	"github.com/arangodb/kube-arangodb/pkg/util/globals"
+	"github.com/rs/zerolog"
 )
 
 type StateInspector interface {
 	RefreshState(ctx context.Context, members api.DeploymentStatusMemberElements)
 	MemberState(id string) (State, bool)
+
+	Log(logger zerolog.Logger)
 }
 
 func NewStateInspector(client reconciler.DeploymentMemberClient) StateInspector {
@@ -48,6 +51,17 @@ type stateInspector struct {
 	client reconciler.DeploymentMemberClient
 }
 
+func (s *stateInspector) Log(logger zerolog.Logger) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	for m, s := range s.members {
+		if s.IsInvalid() {
+			s.Log(logger.Info()).Str("member", m).Msgf("Member is in invalid state")
+		}
+	}
+}
+
 func (s *stateInspector) RefreshState(ctx context.Context, members api.DeploymentStatusMemberElements) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
@@ -58,18 +72,18 @@ func (s *stateInspector) RefreshState(ctx context.Context, members api.Deploymen
 	defer cancel()
 
 	members.ForEach(func(id int) {
+		results[id] = State{}
+
 		c, err := s.client.GetServerClient(nctx, members[id].Group, members[id].Member.ID)
 		if err != nil {
-			results[id].Reachable = false
+			results[id].Reachable = err
 			return
 		}
 
 		if _, err := c.Version(nctx); err != nil {
-			results[id].Reachable = false
+			results[id].Reachable = err
 			return
 		}
-
-		results[id].Reachable = true
 	})
 
 	current := map[string]State{}
@@ -95,5 +109,22 @@ func (s *stateInspector) MemberState(id string) (State, bool) {
 }
 
 type State struct {
-	Reachable bool
+	Reachable error
+}
+
+func (s State) IsReachable() bool {
+	return s.Reachable == nil
+}
+
+func (s State) Log(event *zerolog.Event) *zerolog.Event {
+	if !s.IsReachable() {
+		event = event.Bool("reachable", false).AnErr("reachableError", s.Reachable)
+	} else {
+		event = event.Bool("reachable", false)
+	}
+	return event
+}
+
+func (s State) IsInvalid() bool {
+	return !s.IsReachable()
 }

pkg/deployment/reconcile/action.go

@@ -47,7 +47,7 @@ type Action interface {
 
 	// Timeout returns the amount of time after which this action will timeout.
 	Timeout(deploymentSpec api.DeploymentSpec) time.Duration
-	// Return the MemberID used / created in this action
+	// MemberID Return the MemberID used / created in this action
 	MemberID() string
 }
 
@@ -83,6 +83,22 @@ func getActionReloadCachedStatus(a Action) bool {
 	}
 }
 
+// ActionStartFailureGracePeriod extend action definition to allow specifying start failure grace period
+type ActionStartFailureGracePeriod interface {
+	Action
+
+	// StartFailureGracePeriod returns information about failure grace period (defaults to 0)
+	StartFailureGracePeriod() time.Duration
+}
+
+func getStartFailureGracePeriod(a Action) time.Duration {
+	if c, ok := a.(ActionStartFailureGracePeriod); !ok {
+		return 0
+	} else {
+		return c.StartFailureGracePeriod()
+	}
+}
+
 // ActionPlanAppender modify plan after action execution
 type ActionPlanAppender interface {
 	Action

pkg/deployment/reconcile/action_kill_member_pod.go

@@ -67,6 +67,11 @@ func (a *actionKillMemberPod) Start(ctx context.Context) (bool, error) {
 		return true, nil
 	}
 
+	if ifPodUIDMismatch(m, a.action, a.actionCtx.GetCachedStatus()) {
+		log.Error().Msg("Member UID is changed")
+		return true, nil
+	}
+
 	if err := a.actionCtx.DeletePod(ctx, m.PodName, meta.DeleteOptions{}); err != nil {
 		log.Error().Err(err).Msg("Unable to kill pod")
 		return true, nil

pkg/deployment/reconcile/action_rotate_member.go

@@ -26,6 +26,8 @@ import (
 	"github.com/rs/zerolog"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"time"
+
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
@@ -107,3 +109,7 @@ func (a *actionRotateMember) CheckProgress(ctx context.Context) (bool, bool, err
 	}
 	return true, false, nil
 }
+
+func (a *actionRotateMember) StartFailureGracePeriod() time.Duration {
+	return 30 * time.Second
+}

pkg/deployment/reconcile/helper_shutdown.go

@@ -48,6 +48,11 @@ func getShutdownHelper(a *api.Action, actionCtx ActionContext, log zerolog.Logge
 		return nil, api.MemberStatus{}, false
 	}
 
+	if ifPodUIDMismatch(m, *a, actionCtx.GetCachedStatus()) {
+		log.Error().Msg("Member UID is changed")
+		return NewActionSuccess(), m, true
+	}
+
 	pod, ok := actionCtx.GetCachedStatus().Pod(m.PodName)
 	if !ok {
 		log.Warn().Str("pod-name", m.PodName).Msg("pod is already gone")

pkg/deployment/reconcile/plan_builder_utils.go

@@ -43,8 +43,8 @@ func createRotateMemberPlan(log zerolog.Logger, member api.MemberStatus,
 	plan := api.Plan{
 		api.NewAction(api.ActionTypeCleanTLSKeyfileCertificate, group, member.ID, "Remove server keyfile and enforce renewal/recreation"),
 		api.NewAction(api.ActionTypeResignLeadership, group, member.ID, reason),
-		api.NewAction(api.ActionTypeKillMemberPod, group, member.ID, reason),
-		api.NewAction(api.ActionTypeRotateMember, group, member.ID, reason),
+		withMemberPodUID(member, api.NewAction(api.ActionTypeKillMemberPod, group, member.ID, reason)),
+		withMemberPodUID(member, api.NewAction(api.ActionTypeRotateMember, group, member.ID, reason)),
 		api.NewAction(api.ActionTypeWaitForMemberUp, group, member.ID),
 		api.NewAction(api.ActionTypeWaitForMemberInSync, group, member.ID),
 	}

pkg/deployment/reconcile/plan_executor.go

@@ -175,8 +175,11 @@ func (d *Reconciler) executePlan(ctx context.Context, cachedStatus inspectorInte
 
 		action := d.createAction(log, planAction, cachedStatus)
 
-		done, abort, recall, err := d.executeAction(ctx, log, planAction, action)
+		done, abort, recall, retry, err := d.executeAction(ctx, log, planAction, action)
 		if err != nil {
+			if retry {
+				return plan, true, nil
+			}
 			// The Plan will be cleaned up, so no actions will be in the queue.
 			actionsCurrentPlan.WithLabelValues(d.context.GetName(), planAction.Group.AsRole(), planAction.MemberID,
 				planAction.Type.String(), pg.Type()).Set(0.0)
@@ -247,27 +250,33 @@ func (d *Reconciler) executePlan(ctx context.Context, cachedStatus inspectorInte
 	}
 }
 
-func (d *Reconciler) executeAction(ctx context.Context, log zerolog.Logger, planAction api.Action, action Action) (done, abort, callAgain bool, err error) {
+func (d *Reconciler) executeAction(ctx context.Context, log zerolog.Logger, planAction api.Action, action Action) (done, abort, callAgain, retry bool, err error) {
 	if !planAction.IsStarted() {
 		// Not started yet
 		ready, err := action.Start(ctx)
 		if err != nil {
+			if d := getStartFailureGracePeriod(action); d > 0 && !planAction.CreationTime.IsZero() {
+				if time.Since(planAction.CreationTime.Time) < d {
+					log.Error().Err(err).Msg("Failed to start action, but still in grace period")
+					return false, false, false, true, errors.WithStack(err)
+				}
+			}
 			log.Error().Err(err).Msg("Failed to start action")
-			return false, false, false, errors.WithStack(err)
+			return false, false, false, false, errors.WithStack(err)
 		}
 
 		if ready {
 			log.Debug().Bool("ready", ready).Msg("Action Start completed")
-			return true, false, false, nil
+			return true, false, false, false, nil
 		}
 
-		return false, false, true, nil
+		return false, false, true, false, nil
 	}
 	// First action of plan has been started, check its progress
 	ready, abort, err := action.CheckProgress(ctx)
 	if err != nil {
 		log.Debug().Err(err).Msg("Failed to check action progress")
-		return false, false, false, errors.WithStack(err)
+		return false, false, false, false, errors.WithStack(err)
 	}
 
 	log.Debug().
@@ -276,21 +285,21 @@ func (d *Reconciler) executeAction(ctx context.Context, log zerolog.Logger, plan
 		Msg("Action CheckProgress completed")
 
 	if ready {
-		return true, false, false, nil
+		return true, false, false, false, nil
 	}
 
 	if abort {
 		log.Warn().Msg("Action aborted. Removing the entire plan")
 		d.context.CreateEvent(k8sutil.NewPlanAbortedEvent(d.context.GetAPIObject(), string(planAction.Type), planAction.MemberID, planAction.Group.AsRole()))
-		return false, true, false, nil
+		return false, true, false, false, nil
 	} else if time.Now().After(planAction.CreationTime.Add(action.Timeout(d.context.GetSpec()))) {
 		log.Warn().Msg("Action not finished in time. Removing the entire plan")
 		d.context.CreateEvent(k8sutil.NewPlanTimeoutEvent(d.context.GetAPIObject(), string(planAction.Type), planAction.MemberID, planAction.Group.AsRole()))
-		return false, true, false, nil
+		return false, true, false, false, nil
 	}
 
 	// Timeout not yet expired, come back soon
-	return false, false, true, nil
+	return false, false, true, false, nil
 }
 
 // createAction create action object based on action type

pkg/deployment/reconcile/utils.go

@@ -24,13 +24,16 @@ import (
 	"context"
 	"sort"
 
+	"github.com/arangodb/go-driver"
 	"github.com/arangodb/kube-arangodb/pkg/util/globals"
 
-	"github.com/arangodb/go-driver"
 	core "k8s.io/api/core/v1"
 
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/util"
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/pod"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 func secretKeysToListWithPrefix(s *core.Secret) []string {
@@ -67,3 +70,31 @@ func getCluster(ctx context.Context, planCtx PlanBuilderContext) (driver.Cluster
 
 	return cluster, nil
 }
+
+func withMemberPodUID(m api.MemberStatus, a api.Action) api.Action {
+	if q := m.PodUID; q != "" {
+		return a.AddParam(api.ParamPodUID, string(q))
+	}
+
+	return a
+}
+
+func ifPodUIDMismatch(m api.MemberStatus, a api.Action, i pod.Inspector) bool {
+	ut, ok := a.GetParam(api.ParamPodUID)
+	if !ok || ut == "" {
+		return false
+	}
+
+	u := types.UID(ut)
+
+	if m.PodName == "" {
+		return false
+	}
+
+	p, ok := i.Pod(m.PodName)
+	if !ok {
+		return true
+	}
+
+	return u != p.GetUID()
+}

pkg/deployment/resources/pod_creator.go

@@ -68,15 +68,21 @@ func createArangodArgsWithUpgrade(cachedStatus interfaces.Inspector, input pod.I
 func createArangodArgs(cachedStatus interfaces.Inspector, input pod.Input, additionalOptions ...k8sutil.OptionPair) ([]string, error) {
 	options := k8sutil.CreateOptionPairs(64)
 
-	//scheme := NewURLSchemes(bsCfg.SslKeyFile != "").Arangod
 	scheme := "tcp"
 	if input.Deployment.IsSecure() {
 		scheme = "ssl"
 	}
 
-	options.Addf("--server.endpoint", "%s://%s:%d", scheme, input.Deployment.GetListenAddr(), k8sutil.ArangoPort)
+	if input.GroupSpec.GetExternalPortEnabled() {
+		options.Addf("--server.endpoint", "%s://%s:%d", scheme, input.Deployment.GetListenAddr(), k8sutil.ArangoPort)
+	}
+
 	if port := input.GroupSpec.InternalPort; port != nil {
-		options.Addf("--server.endpoint", "tcp://127.0.0.1:%d", *port)
+		internalScheme := "tcp"
+		if input.Deployment.IsSecure() && input.GroupSpec.InternalPortProtocol.Get() == api.ServerGroupPortProtocolHTTPS {
+			internalScheme = "ssl"
+		}
+		options.Addf("--server.endpoint", "%s://127.0.0.1:%d", internalScheme, *port)
 	}
 
 	// Authentication
@@ -679,6 +685,7 @@ func (r *Resources) EnsurePods(ctx context.Context, cachedStatus inspectorInterf
 	iterator := r.context.GetServerGroupIterator()
 	deploymentStatus, _ := r.context.GetStatus()
 	imageNotFoundOnce := &sync.Once{}
+	changed := false
 
 	if err := iterator.ForeachServerGroup(func(group api.ServerGroup, groupSpec api.ServerGroupSpec, status *api.MemberStatusList) error {
 		for _, m := range *status {
@@ -705,12 +712,20 @@ func (r *Resources) EnsurePods(ctx context.Context, cachedStatus inspectorInterf
 				r.log.Warn().Err(err).Msgf("Ensuring pod failed")
 				return errors.WithStack(err)
 			}
+
+			changed = true
 		}
 		return nil
 	}, &deploymentStatus); err != nil {
 		return errors.WithStack(err)
 	}
 
+	if changed {
+		if err := cachedStatus.Refresh(ctx); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

pkg/deployment/resources/pod_finalizers.go

@@ -41,6 +41,7 @@ import (
 const (
 	podFinalizerRemovedInterval = util.Interval(time.Second / 2)  // Interval used (until new inspection) when Pod finalizers have been removed
 	recheckPodFinalizerInterval = util.Interval(time.Second * 10) // Interval used when Pod finalizers need to be rechecked soon
+	podUnreachableGracePeriod   = time.Second * 15                // Interval used when Pod finalizers need to be rechecked soon
 )
 
 // runPodFinalizers goes through the list of pod finalizers to see if they can be removed.
@@ -52,7 +53,7 @@ func (r *Resources) runPodFinalizers(ctx context.Context, p *v1.Pod, memberStatu
 	// When the main container is terminated, then the whole pod should be terminated,
 	// so sidecar core containers' names should not be checked here.
 	// If Member is not reachable finalizers should be also removed
-	isServerContainerDead := !k8sutil.IsPodServerContainerRunning(p) || util.BoolOrDefault(memberStatus.Conditions.GetValue(api.ConditionTypeReachable), true)
+	isServerContainerDead := !k8sutil.IsPodServerContainerRunning(p) || memberStatus.Conditions.Check(api.ConditionTypeReachable).Exists().IsFalse().LastTransition(podUnreachableGracePeriod).Evaluate()
 
 	for _, f := range p.ObjectMeta.GetFinalizers() {
 		switch f {

pkg/deployment/resources/pod_inspector.go

@@ -219,7 +219,7 @@ func (r *Resources) InspectPods(ctx context.Context, cachedStatus inspectorInter
 		// End of Topology labels
 
 		if state, ok := r.context.MemberState(memberStatus.ID); ok {
-			if state.Reachable {
+			if state.IsReachable() {
 				if memberStatus.Conditions.Update(api.ConditionTypeReachable, true, "ArangoDB is reachable", "") {
 					updateMemberStatusNeeded = true
 					nextInterval = nextInterval.ReduceTo(recheckSoonPodInspectorInterval)