[Feature] [DBServer Maintenance] Agency adjustments (#1076)

CHANGELOG.md

@@ -11,6 +11,7 @@
 - (Bugfix) Check serving servers
 - (Documentation) Add docs on setting timezone for containers
 - (Bugfix) Ensure that client cache is initialized before using it
+- (Feature) (DBServer Maintenance) Agency adjustments
 
 ## [1.2.15](https://github.com/arangodb/kube-arangodb/tree/1.2.15) (2022-07-20)
 - (Bugfix) Ensure pod names not too long

pkg/apis/deployment/v1/conditions.go

@@ -69,6 +69,8 @@ const (
 	// ConditionTypeUpgradeFailed indicates that upgrade failed
 	ConditionTypeUpgradeFailed ConditionType = "UpgradeFailed"
 
+	// ConditionTypeMemberMaintenanceMode indicates that Maintenance is enabled on particular member
+	ConditionTypeMemberMaintenanceMode ConditionType = "MemberMaintenanceMode"
 	// ConditionTypeMaintenanceMode indicates that Maintenance is enabled
 	ConditionTypeMaintenanceMode ConditionType = "MaintenanceMode"
 

pkg/apis/deployment/v1/plan.go

@@ -154,6 +154,10 @@ const (
 	ActionTypeDisableMaintenance ActionType = "DisableMaintenance"
 	// ActionTypeSetMaintenanceCondition sets maintenance condition.
 	ActionTypeSetMaintenanceCondition ActionType = "SetMaintenanceCondition"
+	// ActionTypeEnableMemberMaintenance enables maintenance on cluster member.
+	ActionTypeEnableMemberMaintenance ActionType = "EnableMemberMaintenance"
+	// ActionTypeDisableMemberMaintenance disables maintenance on cluster member.
+	ActionTypeDisableMemberMaintenance ActionType = "DisableMemberMaintenance"
 	// ActionTypeBootstrapUpdate update bootstrap status to true
 	ActionTypeBootstrapUpdate ActionType = "BootstrapUpdate"
 	// ActionTypeBootstrapSetPassword set password to the bootstrapped user

pkg/apis/deployment/v2alpha1/conditions.go

@@ -69,6 +69,8 @@ const (
 	// ConditionTypeUpgradeFailed indicates that upgrade failed
 	ConditionTypeUpgradeFailed ConditionType = "UpgradeFailed"
 
+	// ConditionTypeMemberMaintenanceMode indicates that Maintenance is enabled on particular member
+	ConditionTypeMemberMaintenanceMode ConditionType = "MemberMaintenanceMode"
 	// ConditionTypeMaintenanceMode indicates that Maintenance is enabled
 	ConditionTypeMaintenanceMode ConditionType = "MaintenanceMode"
 

pkg/apis/deployment/v2alpha1/plan.go

@@ -154,6 +154,10 @@ const (
 	ActionTypeDisableMaintenance ActionType = "DisableMaintenance"
 	// ActionTypeSetMaintenanceCondition sets maintenance condition.
 	ActionTypeSetMaintenanceCondition ActionType = "SetMaintenanceCondition"
+	// ActionTypeEnableMemberMaintenance enables maintenance on cluster member.
+	ActionTypeEnableMemberMaintenance ActionType = "EnableMemberMaintenance"
+	// ActionTypeDisableMemberMaintenance disables maintenance on cluster member.
+	ActionTypeDisableMemberMaintenance ActionType = "DisableMemberMaintenance"
 	// ActionTypeBootstrapUpdate update bootstrap status to true
 	ActionTypeBootstrapUpdate ActionType = "BootstrapUpdate"
 	// ActionTypeBootstrapSetPassword set password to the bootstrapped user

pkg/deployment/agency/current_maintenance_servers.go

@@ -0,0 +1,54 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package agency
+
+type StateCurrentMaintenanceServers map[Server]StateCurrentMaintenanceServer
+
+func (s StateCurrentMaintenanceServers) InMaintenance(server Server) bool {
+	if v, ok := s[server]; ok {
+		return v.InMaintenance()
+	}
+
+	return false
+}
+
+type StateCurrentMaintenanceServerMode string
+
+const (
+	StateCurrentMaintenanceServerModeMaintenance StateCurrentMaintenanceServerMode = "maintenance"
+)
+
+type StateCurrentMaintenanceServer struct {
+	Mode  *StateCurrentMaintenanceServerMode `json:"Mode,omitempty"`
+	Until StateTimestamp                     `json:"Until,omitempty"`
+}
+
+func (s *StateCurrentMaintenanceServer) InMaintenance() bool {
+	if s != nil {
+		if m := s.Mode; m != nil {
+			if *m == StateCurrentMaintenanceServerModeMaintenance {
+				return true
+			}
+		}
+	}
+
+	return false
+}

pkg/deployment/agency/definitions.go

@@ -32,6 +32,8 @@ const (
 	CurrentKey = "Current"
 	TargetKey  = "Target"
 
+	CurrentMaintenanceServers = "MaintenanceServers"
+
 	TargetHotBackupKey = "HotBackup"
 
 	PlanCollectionsKey = "Collections"

pkg/deployment/agency/state.go

@@ -45,6 +45,7 @@ func (c *cache) loadState(ctx context.Context, client agency.Agency) (State, err
 		GetAgencyKey(ArangoKey, SupervisionKey, SupervisionMaintenanceKey),
 		GetAgencyKey(ArangoKey, PlanKey, PlanCollectionsKey),
 		GetAgencyKey(ArangoKey, CurrentKey, PlanCollectionsKey),
+		GetAgencyKey(ArangoKey, CurrentKey, CurrentMaintenanceServers),
 		GetAgencyKey(ArangoKey, TargetKey, TargetHotBackupKey),
 		GetAgencyKey(ArangoKey, TargetKey, TargetJobToDoKey),
 		GetAgencyKey(ArangoKey, TargetKey, TargetJobPendingKey),
@@ -100,7 +101,8 @@ type State struct {
 }
 
 type StateCurrent struct {
-	Collections StateCurrentCollections `json:"Collections"`
+	MaintenanceServers StateCurrentMaintenanceServers `json:"MaintenanceServers,omitempty"`
+	Collections        StateCurrentCollections        `json:"Collections"`
 }
 
 type StatePlan struct {

pkg/deployment/client/client.go

@@ -36,6 +36,7 @@ func NewClient(c driver.Connection) Client {
 
 type Client interface {
 	LicenseClient
+	MaintenanceClient
 
 	GetTLS(ctx context.Context) (TLSDetails, error)
 	RefreshTLS(ctx context.Context) (TLSDetails, error)

pkg/deployment/client/maintenance.go

@@ -0,0 +1,89 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package client
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+type MaintenanceClient interface {
+	EnableMaintenance(ctx context.Context, id string, timeout Seconds) error
+	EnableMaintenanceWithDefaultTimeout(ctx context.Context, id string) error
+	DisableMaintenance(ctx context.Context, id string) error
+}
+
+type MemberMaintenanceMode string
+
+const (
+	MemberMaintenanceModeMaintenance MemberMaintenanceMode = "maintenance"
+	MemberMaintenanceModeNormal      MemberMaintenanceMode = "normal"
+
+	MemberMaintenanceUrl = "/_admin/cluster/maintenance/%s"
+
+	DefaultMaintenanceModeTimeout = Seconds(15 * time.Minute)
+)
+
+type MemberMaintenanceRequest struct {
+	Mode    MemberMaintenanceMode `json:"mode"`
+	Timeout *Seconds              `json:"timeout,omitempty"`
+}
+
+func (c *client) EnableMaintenance(ctx context.Context, id string, timeout Seconds) error {
+	return c.setMaintenance(ctx, id, MemberMaintenanceModeMaintenance, timeout.Ptr())
+}
+
+func (c *client) EnableMaintenanceWithDefaultTimeout(ctx context.Context, id string) error {
+	return c.EnableMaintenance(ctx, id, DefaultMaintenanceModeTimeout)
+}
+
+func (c *client) DisableMaintenance(ctx context.Context, id string) error {
+	return c.setMaintenance(ctx, id, MemberMaintenanceModeNormal, nil)
+}
+
+func (c *client) setMaintenance(ctx context.Context, id string, mode MemberMaintenanceMode, timeout *Seconds) error {
+	req, err := c.c.NewRequest(http.MethodPut, fmt.Sprintf(MemberMaintenanceUrl, id))
+	if err != nil {
+		return err
+	}
+
+	if r, err := req.SetBody(MemberMaintenanceRequest{
+		Mode:    mode,
+		Timeout: timeout,
+	}); err != nil {
+		return err
+	} else {
+		req = r
+	}
+
+	resp, err := c.c.Do(ctx, req)
+	if err != nil {
+		return err
+	}
+
+	if err := resp.CheckStatus(http.StatusOK); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/deployment/client/types.go

@@ -0,0 +1,54 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package client
+
+import (
+	"encoding/json"
+	"time"
+)
+
+func SetSeconds(i int) Seconds {
+	return Seconds(time.Second * time.Duration(i))
+}
+
+type Seconds time.Duration
+
+func (s Seconds) Ptr() *Seconds {
+	return &s
+}
+
+func (s *Seconds) UnmarshalJSON(bytes []byte) error {
+	var i int
+
+	if err := json.Unmarshal(bytes, &i); err != nil {
+		return err
+	}
+
+	*s = Seconds(time.Second * time.Duration(i))
+
+	return nil
+}
+
+func (s Seconds) MarshalJSON() ([]byte, error) {
+	i := int(time.Duration(s) / time.Second)
+
+	return json.Marshal(i)
+}

pkg/deployment/features/version_3_10.go

@@ -0,0 +1,37 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package features
+
+func init() {
+	registerFeature(version310)
+}
+
+var version310 = &feature{
+	name:               "version.3-10",
+	description:        "Enable support for 3.10 features",
+	version:            "3.10.0",
+	enterpriseRequired: false,
+	enabledByDefault:   false,
+}
+
+func Version310() Feature {
+	return version310
+}

pkg/deployment/reconcile/action_maintenance_disable.go

@@ -33,7 +33,7 @@ func init() {
 func newDisableMaintenanceAction(action api.Action, actionCtx ActionContext) Action {
 	a := &actionDisableMaintenance{}
 
-	a.actionImpl = newActionImpl(action, actionCtx, &a.newMemberID)
+	a.actionImpl = newActionImplDefRef(action, actionCtx)
 
 	return a
 }
@@ -43,8 +43,6 @@ type actionDisableMaintenance struct {
 	actionImpl
 
 	actionEmptyCheckProgress
-
-	newMemberID string
 }
 
 func (a *actionDisableMaintenance) Start(ctx context.Context) (bool, error) {

pkg/deployment/reconcile/action_maintenance_enable.go

@@ -33,7 +33,7 @@ func init() {
 func newEnableMaintenanceAction(action api.Action, actionCtx ActionContext) Action {
 	a := &actionEnableMaintenance{}
 
-	a.actionImpl = newActionImpl(action, actionCtx, &a.newMemberID)
+	a.actionImpl = newActionImplDefRef(action, actionCtx)
 
 	return a
 }
@@ -43,8 +43,6 @@ type actionEnableMaintenance struct {
 	actionImpl
 
 	actionEmptyCheckProgress
-
-	newMemberID string
 }
 
 func (a *actionEnableMaintenance) Start(ctx context.Context) (bool, error) {

pkg/deployment/reconcile/action_member_maintenance_disable.community.go

@@ -0,0 +1,26 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//go:build !enterprise
+// +build !enterprise
+
+package reconcile
+
+type actionDisableMemberMaintenance struct {
+	actionEmpty
+}

pkg/deployment/reconcile/action_member_maintenance_disable.go

@@ -0,0 +1,37 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package reconcile
+
+import (
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+)
+
+func init() {
+	registerAction(api.ActionTypeDisableMemberMaintenance, newDisableMemberMaintenanceAction, defaultTimeout)
+}
+
+func newDisableMemberMaintenanceAction(action api.Action, actionCtx ActionContext) Action {
+	a := &actionDisableMemberMaintenance{}
+
+	a.actionImpl = newActionImplDefRef(action, actionCtx)
+
+	return a
+}

pkg/deployment/reconcile/action_member_maintenance_enable.community.go

@@ -0,0 +1,26 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//go:build !enterprise
+// +build !enterprise
+
+package reconcile
+
+type actionEnableMemberMaintenance struct {
+	actionEmpty
+}

pkg/deployment/reconcile/action_member_maintenance_enable.go

@@ -0,0 +1,37 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package reconcile
+
+import (
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+)
+
+func init() {
+	registerAction(api.ActionTypeEnableMemberMaintenance, newEnableMemberMaintenanceAction, defaultTimeout)
+}
+
+func newEnableMemberMaintenanceAction(action api.Action, actionCtx ActionContext) Action {
+	a := &actionEnableMemberMaintenance{}
+
+	a.actionImpl = newActionImplDefRef(action, actionCtx)
+
+	return a
+}

pkg/deployment/reconcile/helper_wrap.go

@@ -42,6 +42,24 @@ func withMaintenanceStart(plan ...api.Action) api.Plan {
 		actions.NewClusterAction(api.ActionTypeEnableMaintenance, "Enable maintenance before actions"))
 }
 
+func withMemberMaintenance(group api.ServerGroup, member api.MemberStatus, reason string, plan ...api.Action) api.Plan {
+	if member.Image == nil {
+		return plan
+	}
+
+	if group != api.ServerGroupDBServers {
+		return plan
+	}
+
+	if !features.Version310().Enabled() {
+		return plan
+	}
+
+	return withResignLeadership(group, member, reason, plan...).
+		Wrap(actions.NewAction(api.ActionTypeEnableMemberMaintenance, group, member, reason),
+			actions.NewAction(api.ActionTypeDisableMemberMaintenance, group, member, reason))
+}
+
 func withResignLeadership(group api.ServerGroup, member api.MemberStatus, reason string, plan ...api.Action) api.Plan {
 	if member.Image == nil {
 		return plan

pkg/deployment/reconcile/plan_builder_common.go

@@ -22,11 +22,8 @@ package reconcile
 
 import (
 	"context"
-	"time"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
-	"github.com/arangodb/kube-arangodb/pkg/deployment/actions"
-	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 )
 
@@ -49,57 +46,3 @@ func (r *Reconciler) cleanupConditions(ctx context.Context, apiObject k8sutil.AP
 
 	return p
 }
-
-func (r *Reconciler) createMaintenanceManagementPlan(ctx context.Context, apiObject k8sutil.APIObject,
-	spec api.DeploymentSpec, status api.DeploymentStatus,
-	planCtx PlanBuilderContext) api.Plan {
-	if spec.Mode.Get() == api.DeploymentModeSingle {
-		return nil
-	}
-
-	if !features.Maintenance().Enabled() {
-		// Maintenance feature is not enabled
-		return nil
-	}
-
-	agencyState, agencyOK := planCtx.GetAgencyCache()
-	if !agencyOK {
-		r.log.Error("Unable to get agency mode")
-		return nil
-	}
-
-	if agencyState.Target.HotBackup.Create.Exists() {
-		r.log.Info("HotBackup in progress")
-		return nil
-	}
-
-	enabled := agencyState.Supervision.Maintenance.Exists()
-	c, cok := status.Conditions.Get(api.ConditionTypeMaintenance)
-
-	if (cok && c.IsTrue()) != enabled {
-		// Condition not yet propagated
-		r.log.Info("Condition not yet propagated")
-		return nil
-	}
-
-	if cok {
-		if t := c.LastTransitionTime.Time; !t.IsZero() {
-			if time.Since(t) < 5*time.Second {
-				// Did not elapse 5 s
-				return nil
-			}
-		}
-	}
-
-	if !enabled && spec.Database.GetMaintenance() {
-		r.log.Info("Enabling maintenance mode")
-		return api.Plan{actions.NewClusterAction(api.ActionTypeEnableMaintenance)}
-	}
-
-	if enabled && !spec.Database.GetMaintenance() {
-		r.log.Info("Disabling maintenance mode")
-		return api.Plan{actions.NewClusterAction(api.ActionTypeDisableMaintenance)}
-	}
-
-	return nil
-}

pkg/deployment/reconcile/plan_builder_maintenance.community.go

@@ -0,0 +1,35 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//go:build !enterprise
+// +build !enterprise
+
+package reconcile
+
+import (
+	"context"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+)
+
+func (r *Reconciler) createMemberMaintenanceManagementPlan(ctx context.Context, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	planCtx PlanBuilderContext) api.Plan {
+	return nil
+}

pkg/deployment/reconcile/plan_builder_maintenance.go

@@ -22,8 +22,11 @@ package reconcile
 
 import (
 	"context"
+	"time"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/actions"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 )
 
@@ -122,3 +125,57 @@ func (r *Reconciler) createMaintenanceConditionPlan(ctx context.Context, apiObje
 		return nil
 	}
 }
+
+func (r *Reconciler) createMaintenanceManagementPlan(ctx context.Context, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	planCtx PlanBuilderContext) api.Plan {
+	if spec.Mode.Get() == api.DeploymentModeSingle {
+		return nil
+	}
+
+	if !features.Maintenance().Enabled() {
+		// Maintenance feature is not enabled
+		return nil
+	}
+
+	agencyState, agencyOK := planCtx.GetAgencyCache()
+	if !agencyOK {
+		r.log.Error("Unable to get agency mode")
+		return nil
+	}
+
+	if agencyState.Target.HotBackup.Create.Exists() {
+		r.log.Info("HotBackup in progress")
+		return nil
+	}
+
+	enabled := agencyState.Supervision.Maintenance.Exists()
+	c, cok := status.Conditions.Get(api.ConditionTypeMaintenance)
+
+	if (cok && c.IsTrue()) != enabled {
+		// Condition not yet propagated
+		r.log.Info("Condition not yet propagated")
+		return nil
+	}
+
+	if cok {
+		if t := c.LastTransitionTime.Time; !t.IsZero() {
+			if time.Since(t) < 5*time.Second {
+				// Did not elapse 5 s
+				return nil
+			}
+		}
+	}
+
+	if !enabled && spec.Database.GetMaintenance() {
+		r.log.Info("Enabling maintenance mode")
+		return api.Plan{actions.NewClusterAction(api.ActionTypeEnableMaintenance)}
+	}
+
+	if enabled && !spec.Database.GetMaintenance() {
+		r.log.Info("Disabling maintenance mode")
+		return api.Plan{actions.NewClusterAction(api.ActionTypeDisableMaintenance)}
+	}
+
+	return nil
+}

pkg/deployment/reconcile/plan_builder_normal.go

@@ -61,6 +61,7 @@ func (r *Reconciler) createNormalPlan(ctx context.Context, apiObject k8sutil.API
 		ApplyIfEmpty(r.createReplaceMemberPlan).
 		// Check for the need to rotate one or more members
 		ApplyIfEmpty(r.createMarkToRemovePlan).
+		ApplyIfEmpty(r.createMemberMaintenanceManagementPlan).
 		ApplyIfEmpty(r.createRotateOrUpgradePlan).
 		// Disable maintenance if upgrade process was done. Upgrade task throw IDLE Action if upgrade is pending
 		ApplyIfEmpty(r.createMaintenanceManagementPlan).

pkg/deployment/reconcile/plan_builder_rotate_upgrade.go

@@ -524,7 +524,7 @@ func withSecureWrap(member api.MemberStatus,
 		// In this case we skip resign leadership but we enable maintenance
 		return withMaintenanceStart(plan...)
 	} else {
-		return withResignLeadership(group, member, "ResignLeadership", plan...)
+		return withMemberMaintenance(group, member, "ResignLeadership", plan...)
 	}
 }
 

pkg/deployment/resources/pod_inspector.go

@@ -30,6 +30,7 @@ import (
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/agency"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/patch"
 	"github.com/arangodb/kube-arangodb/pkg/metrics"
 	"github.com/arangodb/kube-arangodb/pkg/util"
@@ -87,6 +88,8 @@ func (r *Resources) InspectPods(ctx context.Context, cachedStatus inspectorInter
 	nextInterval := maxPodInspectorInterval // Large by default, will be made smaller if needed in the rest of the function
 	defer metrics.SetDuration(inspectPodsDurationGauges.WithLabelValues(deploymentName), start)
 
+	agencyCache, agencyCachePresent := r.context.GetAgencyCache()
+
 	status, lastVersion := r.context.GetStatus()
 	var podNamesWithScheduleTimeout []string
 	var unscheduledPodNames []string
@@ -270,6 +273,21 @@ func (r *Resources) InspectPods(ctx context.Context, cachedStatus inspectorInter
 			}
 		}
 
+		// Member Maintenance
+		if agencyCachePresent {
+			if agencyCache.Current.MaintenanceServers.InMaintenance(agency.Server(memberStatus.ID)) {
+				if memberStatus.Conditions.Update(api.ConditionTypeMemberMaintenanceMode, true, "ArangoDB Member maintenance enabled", "") {
+					updateMemberStatusNeeded = true
+					nextInterval = nextInterval.ReduceTo(recheckSoonPodInspectorInterval)
+				}
+			} else {
+				if memberStatus.Conditions.Remove(api.ConditionTypeMemberMaintenanceMode) {
+					updateMemberStatusNeeded = true
+					nextInterval = nextInterval.ReduceTo(recheckSoonPodInspectorInterval)
+				}
+			}
+		}
+
 		if k8sutil.IsPodReady(pod) && k8sutil.AreContainersReady(pod, coreContainers) {
 			// Pod is now ready
 			if anyOf(memberStatus.Conditions.Update(api.ConditionTypeReady, true, "Pod Ready", ""),

pkg/deployment/versions/check.go

@@ -0,0 +1,133 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package versions
+
+import (
+	"github.com/arangodb/go-driver"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+)
+
+func NewCheck(version api.ImageInfo) Check {
+	return check{
+		version: version,
+	}
+}
+
+type Check interface {
+	Enterprise() Check
+	Community() Check
+
+	Above(version driver.Version) Check
+	AboveOrEqual(version driver.Version) Check
+
+	Below(version driver.Version) Check
+	BelowOrEqual(version driver.Version) Check
+
+	Evaluate() bool
+}
+
+type check struct {
+	version api.ImageInfo
+}
+
+func (c check) Below(version driver.Version) Check {
+	if c.version.ArangoDBVersion.CompareTo(version) == 1 {
+		return c
+	}
+
+	return falseCheck{}
+}
+
+func (c check) BelowOrEqual(version driver.Version) Check {
+	if c.version.ArangoDBVersion.CompareTo(version) <= 0 {
+		return c
+	}
+
+	return falseCheck{}
+}
+
+func (c check) Above(version driver.Version) Check {
+	if c.version.ArangoDBVersion.CompareTo(version) == -1 {
+		return c
+	}
+
+	return falseCheck{}
+}
+
+func (c check) AboveOrEqual(version driver.Version) Check {
+	if c.version.ArangoDBVersion.CompareTo(version) >= 0 {
+		return c
+	}
+
+	return falseCheck{}
+}
+
+func (c check) Enterprise() Check {
+	if c.version.Enterprise {
+		return c
+	}
+
+	return falseCheck{}
+}
+
+func (c check) Community() Check {
+	if !c.version.Enterprise {
+		return c
+	}
+
+	return falseCheck{}
+}
+
+func (c check) Evaluate() bool {
+	return true
+}
+
+type falseCheck struct {
+}
+
+func (f falseCheck) Below(version driver.Version) Check {
+	return f
+}
+
+func (f falseCheck) BelowOrEqual(version driver.Version) Check {
+	return f
+}
+
+func (f falseCheck) Above(version driver.Version) Check {
+	return f
+}
+
+func (f falseCheck) AboveOrEqual(version driver.Version) Check {
+	return f
+}
+
+func (f falseCheck) Enterprise() Check {
+	return f
+}
+
+func (f falseCheck) Community() Check {
+	return f
+}
+
+func (f falseCheck) Evaluate() bool {
+	return false
+}

pkg/deployment/versions/check_test.go

@@ -0,0 +1,84 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package versions
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/arangodb/go-driver"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+)
+
+func runCheckTest(t *testing.T, name string, version driver.Version, enterprise bool, expected bool, checker ...func(c Check) Check) {
+	t.Run(name, func(t *testing.T) {
+		c := NewCheck(api.ImageInfo{
+			ArangoDBVersion: version,
+			Enterprise:      enterprise,
+		})
+
+		for _, f := range checker {
+			c = f(c)
+		}
+
+		if expected {
+			require.True(t, c.Evaluate())
+		} else {
+			require.False(t, c.Evaluate())
+		}
+	})
+}
+
+func Test_Check(t *testing.T) {
+	runCheckTest(t, "Empty", "3.6.0", true, true)
+
+	runCheckTest(t, "Enterprise - ok", "3.6.0", true, true, func(c Check) Check {
+		return c.Enterprise()
+	})
+	runCheckTest(t, "Enterprise - invalid", "3.6.0", false, false, func(c Check) Check {
+		return c.Enterprise()
+	})
+
+	runCheckTest(t, "Community - ok", "3.6.0", false, true, func(c Check) Check {
+		return c.Community()
+	})
+	runCheckTest(t, "Community - invalid", "3.6.0", true, false, func(c Check) Check {
+		return c.Community()
+	})
+
+	runCheckTest(t, "3.6.5 <=", "3.6.5", true, true, func(c Check) Check {
+		return c.AboveOrEqual("3.6.5")
+	})
+
+	runCheckTest(t, "3.6.5 <", "3.6.5", true, false, func(c Check) Check {
+		return c.Above("3.6.5")
+	})
+
+	runCheckTest(t, "3.6.5 >=", "3.6.5", true, true, func(c Check) Check {
+		return c.AboveOrEqual("3.6.5")
+	})
+
+	runCheckTest(t, "3.6.5 >", "3.6.5", true, false, func(c Check) Check {
+		return c.Above("3.6.5")
+	})
+}

pkg/deployment/versions/member_maintenance.go

@@ -0,0 +1,31 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package versions
+
+import api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+
+func MemberMaintenance(in api.ImageInfo) bool {
+	return memberMaintenanceChecker(NewCheck(in)).Evaluate()
+}
+
+func memberMaintenanceChecker(in Check) Check {
+	return in.AboveOrEqual("3.10.0")
+}

pkg/deployment/versions/member_maintenance_test.go

@@ -0,0 +1,38 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package versions
+
+import (
+	"testing"
+)
+
+func Test_MemberMaintenance(t *testing.T) {
+	runCheckTest(t, "MemberMaintenance - EE - 3.10", "3.10.0", true, true, memberMaintenanceChecker)
+	runCheckTest(t, "MemberMaintenance - CE - 3.10", "3.10.0", false, true, memberMaintenanceChecker)
+	runCheckTest(t, "MemberMaintenance - EE - 3.10.1", "3.10.1", true, true, memberMaintenanceChecker)
+	runCheckTest(t, "MemberMaintenance - CE - 3.10.1", "3.10.1", false, true, memberMaintenanceChecker)
+	runCheckTest(t, "MemberMaintenance - EE - 3.11", "3.11.0", true, true, memberMaintenanceChecker)
+	runCheckTest(t, "MemberMaintenance - CE - 3.11", "3.11.0", false, true, memberMaintenanceChecker)
+	runCheckTest(t, "MemberMaintenance - EE - 3.9", "3.9.0", true, false, memberMaintenanceChecker)
+	runCheckTest(t, "MemberMaintenance - CE - 3.9", "3.9.0", false, false, memberMaintenanceChecker)
+	runCheckTest(t, "MemberMaintenance - EE - 4.0", "4.0.0", true, true, memberMaintenanceChecker)
+	runCheckTest(t, "MemberMaintenance - CE - 4.0", "4.0.0", false, true, memberMaintenanceChecker)
+}

pkg/util/k8sutil/license_test.go

@@ -73,16 +73,28 @@ func generateLK(t *testing.T) string {
 	return string(i)
 }
 
+func Test_GetLicensePredefined(t *testing.T) {
+	lk := "{\"grant\":\"eyJmZWF0dXJlcyI6eyJleHBpcmVzIjoxNjU5MDc3OTk5fSwibm90QmVmb3JlIjoiMjAyMi0wNy0yN1QxMDowOTozNFoiLCJub3RBZnRlciI6IjIwMjItMDctMjlUMDY6NTk6NTlaIiwic3ViamVjdCI6eyJlbWFpbCI6ImdvdGVhbUBhcmFuZ29kYi5jb20ifSwiaXNzdWVyIjp7ImNvbXBhbnkiOiJBcmFuZ29EQiBHbWJIIiwibmFtZSI6IlFBIChBZGFtKSIsImVtYWlsIjoiYWRhbUBhcmFuZ29kYi5jb20ifSwidmVyc2lvbiI6MX0=\",\"signature\":\"F1jGVQ6nuqQJidGzAeNpzbLmTF6J9cv/sPcEUiR0uxY9202PDC7jfqGSgyrAvEXXX5HaLAsRsf2w0/wgkO5KroaJGlONqVKyFkCL2gYSHL1cZl1O9Zt128qAvUL1L5xGeDXEaS9XC7H3DvLecdAYF9URUumytFfUvHUH9jAPo1k8U2WF7sG3c1z0TlPOJbHo5Z0uy2rlcAEiuCKdrZgWXvBcYYnAaBNQIC8HJwTlqMnUI9NY8qm8St8L6lPHTv6cRl0OBW+Z/+3DmZoOW/j0jZNAE57b8DmwUHsdVDtFBhsl4u62jCD1qcRxuYeQ1QTG8+mFzsB0I5MMj4obSeSYOg==\"}"
+	lke := "eyJncmFudCI6ImV5Sm1aV0YwZFhKbGN5STZleUpsZUhCcGNtVnpJam94TmpVNU1EYzNPVGs1ZlN3aWJtOTBRbVZtYjNKbElqb2lNakF5TWkwd055MHlOMVF4TURvd09Ub3pORm9pTENKdWIzUkJablJsY2lJNklqSXdNakl0TURjdE1qbFVNRFk2TlRrNk5UbGFJaXdpYzNWaWFtVmpkQ0k2ZXlKbGJXRnBiQ0k2SW1kdmRHVmhiVUJoY21GdVoyOWtZaTVqYjIwaWZTd2lhWE56ZFdWeUlqcDdJbU52YlhCaGJua2lPaUpCY21GdVoyOUVRaUJIYldKSUlpd2libUZ0WlNJNklsRkJJQ2hCWkdGdEtTSXNJbVZ0WVdsc0lqb2lZV1JoYlVCaGNtRnVaMjlrWWk1amIyMGlmU3dpZG1WeWMybHZiaUk2TVgwPSIsInNpZ25hdHVyZSI6IkYxakdWUTZudXFRSmlkR3pBZU5wemJMbVRGNko5Y3Yvc1BjRVVpUjB1eFk5MjAyUERDN2pmcUdTZ3lyQXZFWFhYNUhhTEFzUnNmMncwL3dna081S3JvYUpHbE9OcVZLeUZrQ0wyZ1lTSEwxY1psMU85WnQxMjhxQXZVTDFMNXhHZURYRWFTOVhDN0gzRHZMZWNkQVlGOVVSVXVteXRGZlV2SFVIOWpBUG8xazhVMldGN3NHM2MxejBUbFBPSmJIbzVaMHV5MnJsY0FFaXVDS2RyWmdXWHZCY1lZbkFhQk5RSUM4SEp3VGxxTW5VSTlOWThxbThTdDhMNmxQSFR2NmNSbDBPQlcrWi8rM0RtWm9PVy9qMGpaTkFFNTdiOERtd1VIc2RWRHRGQmhzbDR1NjJqQ0QxcWNSeHVZZVExUVRHOCttRnpzQjBJNU1NajRvYlNlU1lPZz09In0="
+
+	getLicenseFromSecret(t, lk, lke)
+}
+
 func Test_GetLicenseFromSecret(t *testing.T) {
 	lk := generateLK(t)
 	lke := encodeLicenseKey(lk)
 
+	getLicenseFromSecret(t, lk, lke)
+}
+
+func getLicenseFromSecret(t *testing.T, raw, encoded string) {
+
 	c := kclient.NewFakeClient()
 	i := tests.NewInspector(t, c)
 
 	t.Run(constants.SecretKeyV2License, func(t *testing.T) {
 		t.Run("Encoded license", func(t *testing.T) {
-			n := createLicenseSecret(t, c, constants.SecretKeyV2License, lke)
+			n := createLicenseSecret(t, c, constants.SecretKeyV2License, encoded)
 
 			require.NoError(t, i.Refresh(context.Background()))
 
@@ -91,11 +103,11 @@ func Test_GetLicenseFromSecret(t *testing.T) {
 
 			require.Empty(t, license.V1)
 			require.NotEmpty(t, license.V2)
-			require.EqualValues(t, lke, license.V2)
+			require.EqualValues(t, encoded, license.V2)
 		})
 
 		t.Run("Raw license", func(t *testing.T) {
-			n := createLicenseSecret(t, c, constants.SecretKeyV2License, lk)
+			n := createLicenseSecret(t, c, constants.SecretKeyV2License, raw)
 
 			require.NoError(t, i.Refresh(context.Background()))
 
@@ -104,13 +116,13 @@ func Test_GetLicenseFromSecret(t *testing.T) {
 
 			require.Empty(t, license.V1)
 			require.NotEmpty(t, license.V2)
-			require.EqualValues(t, lke, license.V2)
+			require.EqualValues(t, encoded, license.V2)
 		})
 	})
 
 	t.Run(constants.SecretKeyV2Token, func(t *testing.T) {
 		t.Run("Encoded license", func(t *testing.T) {
-			n := createLicenseSecret(t, c, constants.SecretKeyV2Token, lke)
+			n := createLicenseSecret(t, c, constants.SecretKeyV2Token, encoded)
 
 			require.NoError(t, i.Refresh(context.Background()))
 
@@ -119,11 +131,11 @@ func Test_GetLicenseFromSecret(t *testing.T) {
 
 			require.Empty(t, license.V1)
 			require.NotEmpty(t, license.V2)
-			require.EqualValues(t, lke, license.V2)
+			require.EqualValues(t, encoded, license.V2)
 		})
 
 		t.Run("Raw license", func(t *testing.T) {
-			n := createLicenseSecret(t, c, constants.SecretKeyV2Token, lk)
+			n := createLicenseSecret(t, c, constants.SecretKeyV2Token, raw)
 
 			require.NoError(t, i.Refresh(context.Background()))
 
@@ -132,7 +144,7 @@ func Test_GetLicenseFromSecret(t *testing.T) {
 
 			require.Empty(t, license.V1)
 			require.NotEmpty(t, license.V2)
-			require.EqualValues(t, lke, license.V2)
+			require.EqualValues(t, encoded, license.V2)
 		})
 	})
 }