[Feature] Add Maintenance Mode support (#623)

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Change Log
 
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
+- Add Operator Maintenance Management feature
 
 ## [1.0.6](https://github.com/arangodb/kube-arangodb/tree/1.0.6) (2020-08-19)
 - Add Operator Namespaced mode (Alpha)

README.md

@@ -60,25 +60,26 @@ covers individual newer features separately.
 
 Feature-wise production readiness table:
 
-| Feature                         | Operator Version | ArangoDB Version | ArangoDB Edition      | State      | Enabled | Flag                                     | Remarks                                                                  |
-|---------------------------------|------------------|------------------|-----------------------|------------|---------|------------------------------------------|--------------------------------------------------------------------------|
-| Pod Disruption Budgets          | 0.3.10           | Any              | Community, Enterprise | Alpha      | True    | N/A                                      | N/A                                                                      |
-| Pod Disruption Budgets          | 0.3.11           | Any              | Community, Enterprise | Production | True    | N/A                                      | N/A                                                                      |
-| Volume Resizing                 | 0.3.10           | Any              | Community, Enterprise | Alpha      | True    | N/A                                      | N/A                                                                      |
-| Volume Resizing                 | 0.3.11           | Any              | Community, Enterprise | Production | True    | N/A                                      | N/A                                                                      |
-| Disabling of liveness probes    | 0.3.10           | Any              | Community, Enterprise | Alpha      | True    | N/A                                      | N/A                                                                      |
-| Disabling of liveness probes    | 0.3.11           | Any              | Community, Enterprise | Production | True    | N/A                                      | N/A                                                                      |
-| Volume Claim Templates          | 0.3.11           | Any              | Community, Enterprise | Alpha      | True    | N/A                                      | N/A                                                                      |
-| Volume Claim Templates          | 1.0.0            | Any              | Community, Enterprise | Production | True    | N/A                                      | N/A                                                                      |
-| Prometheus Metrics Exporter     | 0.3.11           | Any              | Community, Enterprise | Alpha      | True    | N/A                                      | Prometheus required                                                      |
-| Prometheus Metrics Exporter     | 1.0.0            | Any              | Community, Enterprise | Production | True    | N/A                                      | Prometheus required                                                      |
-| Sidecar Containers              | 0.3.11           | Any              | Community, Enterprise | Alpha      | True    | N/A                                      | N/A                                                                      |
-| Sidecar Containers              | 1.0.0            | Any              | Community, Enterprise | Production | True    | N/A                                      | N/A                                                                      |
-| Operator Single Mode            | 1.0.4            | Any              | Community, Enterprise | Production | False   | --mode.single                            | Only 1 instance of Operator allowed in namespace when feature is enabled |
-| TLS SNI Support                 | 1.0.3            | >= 3.7.0         | Enterprise            | Production | True    | --deployment.feature.tls-sni             | N/A                                                                      |
-| TLS Runtime Rotation Support    | 1.0.4            | > 3.7.0          | Enterprise            | Alpha      | False   | --deployment.feature.tls-rotation        | N/A                                                                      |
-| JWT Rotation Support            | 1.0.4            | > 3.7.0          | Enterprise            | Alpha      | False   | --deployment.feature.jwt-rotation        | N/A                                                                      |
-| Encryption Key Rotation Support | 1.0.4            | > 3.7.0          | Enterprise            | Alpha      | False   | --deployment.feature.encryption-rotation | N/A                                                                      |
+| Feature                                 | Operator Version | ArangoDB Version | ArangoDB Edition      | State      | Enabled | Flag                                     | Remarks                                                                  |
+|-----------------------------------------|------------------|------------------|-----------------------|------------|---------|------------------------------------------|--------------------------------------------------------------------------|
+| Pod Disruption Budgets                  | 0.3.10           | Any              | Community, Enterprise | Alpha      | True    | N/A                                      | N/A                                                                      |
+| Pod Disruption Budgets                  | 0.3.11           | Any              | Community, Enterprise | Production | True    | N/A                                      | N/A                                                                      |
+| Volume Resizing                         | 0.3.10           | Any              | Community, Enterprise | Alpha      | True    | N/A                                      | N/A                                                                      |
+| Volume Resizing                         | 0.3.11           | Any              | Community, Enterprise | Production | True    | N/A                                      | N/A                                                                      |
+| Disabling of liveness probes            | 0.3.10           | Any              | Community, Enterprise | Alpha      | True    | N/A                                      | N/A                                                                      |
+| Disabling of liveness probes            | 0.3.11           | Any              | Community, Enterprise | Production | True    | N/A                                      | N/A                                                                      |
+| Volume Claim Templates                  | 0.3.11           | Any              | Community, Enterprise | Alpha      | True    | N/A                                      | N/A                                                                      |
+| Volume Claim Templates                  | 1.0.0            | Any              | Community, Enterprise | Production | True    | N/A                                      | N/A                                                                      |
+| Prometheus Metrics Exporter             | 0.3.11           | Any              | Community, Enterprise | Alpha      | True    | N/A                                      | Prometheus required                                                      |
+| Prometheus Metrics Exporter             | 1.0.0            | Any              | Community, Enterprise | Production | True    | N/A                                      | Prometheus required                                                      |
+| Sidecar Containers                      | 0.3.11           | Any              | Community, Enterprise | Alpha      | True    | N/A                                      | N/A                                                                      |
+| Sidecar Containers                      | 1.0.0            | Any              | Community, Enterprise | Production | True    | N/A                                      | N/A                                                                      |
+| Operator Single Mode                    | 1.0.4            | Any              | Community, Enterprise | Production | False   | --mode.single                            | Only 1 instance of Operator allowed in namespace when feature is enabled |
+| TLS SNI Support                         | 1.0.3            | >= 3.7.0         | Enterprise            | Production | True    | --deployment.feature.tls-sni             | N/A                                                                      |
+| TLS Runtime Rotation Support            | 1.0.4            | > 3.7.0          | Enterprise            | Alpha      | False   | --deployment.feature.tls-rotation        | N/A                                                                      |
+| JWT Rotation Support                    | 1.0.4            | > 3.7.0          | Enterprise            | Alpha      | False   | --deployment.feature.jwt-rotation        | N/A                                                                      |
+| Encryption Key Rotation Support         | 1.0.4            | > 3.7.0          | Enterprise            | Alpha      | False   | --deployment.feature.encryption-rotation | N/A                                                                      |
+| Operator Maintenance Management Support | 1.0.7            | >= 3.5.0         | Community, Enterprise | Alpha      | False   | --deployment.feature.maintenance         | N/A                                                                      |
 
 ## Release notes for 0.3.16
 

pkg/apis/deployment/v1/database_spec.go

@@ -0,0 +1,35 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package v1
+
+type DatabaseSpec struct {
+	Maintenance *bool `json:"maintenance,omitempty"`
+}
+
+func (m *DatabaseSpec) GetMaintenance() bool {
+	if m == nil || m.Maintenance == nil {
+		return false
+	}
+
+	return *m.Maintenance
+}

pkg/apis/deployment/v1/deployment_spec.go

@@ -95,6 +95,8 @@ type DeploymentSpec struct {
 
 	ID *ServerIDGroupSpec `json:"id,omitempty"`
 
+	Database *DatabaseSpec `json:"database,omitempty"`
+
 	Single       ServerGroupSpec `json:"single"`
 	Agents       ServerGroupSpec `json:"agents"`
 	DBServers    ServerGroupSpec `json:"dbservers"`
@@ -295,6 +297,9 @@ func (s *DeploymentSpec) SetDefaultsFrom(source DeploymentSpec) {
 	if s.AllowUnsafeUpgrade == nil {
 		s.AllowUnsafeUpgrade = util.NewBoolOrNil(source.AllowUnsafeUpgrade)
 	}
+	if s.Database == nil {
+		s.Database = source.Database.DeepCopy()
+	}
 
 	s.License.SetDefaultsFrom(source.License)
 	s.ExternalAccess.SetDefaultsFrom(source.ExternalAccess)

pkg/apis/deployment/v1/plan.go

@@ -119,6 +119,10 @@ const (
 	ActionTypeJWTPropagated ActionType = "JWTPropagated"
 	// ActionTypeClusterMemberCleanup removes member from cluster
 	ActionTypeClusterMemberCleanup ActionType = "ClusterMemberCleanup"
+	// ActionTypeEnableMaintenance enables maintenance on cluster.
+	ActionTypeEnableMaintenance ActionType = "EnableMaintenance"
+	// ActionTypeEnableMaintenance disables maintenance on cluster.
+	ActionTypeDisableMaintenance ActionType = "DisableMaintenance"
 )
 
 const (

pkg/apis/deployment/v1/zz_generated.deepcopy.go

@@ -258,6 +258,27 @@ func (in ConditionList) DeepCopy() ConditionList {
 	return *out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DatabaseSpec) DeepCopyInto(out *DatabaseSpec) {
+	*out = *in
+	if in.Maintenance != nil {
+		in, out := &in.Maintenance, &out.Maintenance
+		*out = new(bool)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DatabaseSpec.
+func (in *DatabaseSpec) DeepCopy() *DatabaseSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(DatabaseSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentFeatures) DeepCopyInto(out *DeploymentFeatures) {
 	*out = *in
@@ -415,6 +436,11 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 		*out = new(ServerIDGroupSpec)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Database != nil {
+		in, out := &in.Database, &out.Database
+		*out = new(DatabaseSpec)
+		(*in).DeepCopyInto(*out)
+	}
 	in.Single.DeepCopyInto(&out.Single)
 	in.Agents.DeepCopyInto(&out.Agents)
 	in.DBServers.DeepCopyInto(&out.DBServers)

pkg/deployment/agency/agency.go

@@ -22,41 +22,25 @@
 
 package agency
 
-type ArangoPlanDatabases map[string]ArangoPlanCollections
+import (
+	"context"
 
-func (a ArangoPlanDatabases) IsDBServerInDatabases(name string) bool {
-	for _, collections := range a {
-		if collections.IsDBServerInCollections(name) {
-			return true
+	"github.com/arangodb/go-driver/agency"
+	"github.com/pkg/errors"
+)
+
+type Fetcher func(ctx context.Context, i interface{}, keyParts ...string) error
+
+func NewFetcher(a agency.Agency) Fetcher {
+	return func(ctx context.Context, i interface{}, keyParts ...string) error {
+		if err := a.ReadKey(ctx, []string{
+			ArangoKey,
+			PlanKey,
+			PlanCollectionsKey,
+		}, i); err != nil {
+			return errors.WithStack(err)
 		}
+
+		return nil
 	}
-	return false
 }
-
-type ArangoPlanCollections map[string]ArangoPlanCollection
-
-func (a ArangoPlanCollections) IsDBServerInCollections(name string) bool {
-	for _, collection := range a {
-		if collection.IsDBServerInShards(name) {
-			return true
-		}
-	}
-	return false
-}
-
-type ArangoPlanCollection struct {
-	Shards ArangoPlanShard `json:"shards"`
-}
-
-func (a ArangoPlanCollection) IsDBServerInShards(name string) bool {
-	for _, dbservers := range a.Shards {
-		for _, dbserver := range dbservers {
-			if dbserver == name {
-				return true
-			}
-		}
-	}
-	return false
-}
-
-type ArangoPlanShard map[string][]string

pkg/deployment/agency/database.go

@@ -0,0 +1,78 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package agency
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+)
+
+func GetAgencyCollections(ctx context.Context, f Fetcher) (*ArangoPlanDatabases, error) {
+	ret := &ArangoPlanDatabases{}
+
+	if err := f(ctx, ret, ArangoKey, PlanKey, PlanCollectionsKey); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return ret, nil
+}
+
+type ArangoPlanDatabases map[string]ArangoPlanCollections
+
+func (a ArangoPlanDatabases) IsDBServerInDatabases(name string) bool {
+	for _, collections := range a {
+		if collections.IsDBServerInCollections(name) {
+			return true
+		}
+	}
+	return false
+}
+
+type ArangoPlanCollections map[string]ArangoPlanCollection
+
+func (a ArangoPlanCollections) IsDBServerInCollections(name string) bool {
+	for _, collection := range a {
+		if collection.IsDBServerInShards(name) {
+			return true
+		}
+	}
+	return false
+}
+
+type ArangoPlanCollection struct {
+	Shards ArangoPlanShard `json:"shards"`
+}
+
+func (a ArangoPlanCollection) IsDBServerInShards(name string) bool {
+	for _, dbservers := range a.Shards {
+		for _, dbserver := range dbservers {
+			if dbserver == name {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+type ArangoPlanShard map[string][]string

pkg/deployment/agency/maintenance.go

@@ -0,0 +1,91 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package agency
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/arangodb/go-driver"
+)
+
+type Maintenance struct {
+	Result string `json:"result"`
+}
+
+func (m Maintenance) Enabled() bool {
+	return m.Result == "Maintenance"
+}
+
+func GetMaintenanceMode(ctx context.Context, client driver.Client) (Maintenance, error) {
+	conn := client.Connection()
+	r, err := conn.NewRequest(http.MethodGet, "/_admin/cluster/maintenance")
+	if err != nil {
+		return Maintenance{}, err
+	}
+
+	resp, err := conn.Do(ctx, r)
+	if err != nil {
+		return Maintenance{}, err
+	}
+
+	if err := resp.CheckStatus(http.StatusOK); err != nil {
+		return Maintenance{}, err
+	}
+
+	var m Maintenance
+
+	if err := resp.ParseBody("", &m); err != nil {
+		return Maintenance{}, err
+	}
+
+	return m, nil
+}
+
+func SetMaintenanceMode(ctx context.Context, client driver.Client, enabled bool) error {
+	data := "on"
+	if !enabled {
+		data = "off"
+	}
+
+	conn := client.Connection()
+	r, err := conn.NewRequest(http.MethodPut, "/_admin/cluster/maintenance")
+	if err != nil {
+		return err
+	}
+
+	if _, err := r.SetBody(data); err != nil {
+		return err
+	}
+
+	resp, err := conn.Do(ctx, r)
+	if err != nil {
+		return err
+	}
+
+	if err := resp.CheckStatus(http.StatusOK); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/deployment/features/encryption.go

@@ -17,6 +17,8 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
+// Author Adam Janikowski
+//
 
 package features
 

pkg/deployment/features/features.go

@@ -17,6 +17,8 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
+// Author Adam Janikowski
+//
 
 package features
 

pkg/deployment/features/jwt.go

@@ -17,6 +17,8 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
+// Author Adam Janikowski
+//
 
 package features
 

pkg/deployment/features/local.go

@@ -17,6 +17,8 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
+// Author Adam Janikowski
+//
 
 package features
 
@@ -70,7 +72,7 @@ func Init(cmd *cobra.Command) {
 			if v != "" && feature.EnterpriseRequired() {
 				z = fmt.Sprintf("%s - Required version %s and Enterprise Edition", feature.Description(), v)
 			} else if v != "" {
-				z = fmt.Sprintf("%s. Required version %s", feature.Description(), v)
+				z = fmt.Sprintf("%s - Required version %s", feature.Description(), v)
 			} else if feature.EnterpriseRequired() {
 				z = fmt.Sprintf("%s - Required Enterprise Edition", feature.Description())
 			} else {

pkg/deployment/features/maintenance.go

@@ -0,0 +1,39 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package features
+
+func init() {
+	registerFeature(maintenance)
+}
+
+var maintenance = &feature{
+	name:               "maintenance",
+	description:        "Database maintenance mode management",
+	version:            "3.5.0",
+	enterpriseRequired: false,
+	enabledByDefault:   false,
+}
+
+func Maintenance() Feature {
+	return maintenance
+}

pkg/deployment/features/tls.go

@@ -17,6 +17,8 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
+// Author Adam Janikowski
+//
 
 package features
 

pkg/deployment/reconcile/action_disable_scaling_cluster.go

@@ -17,6 +17,8 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
+// Author Adam Janikowski
+//
 
 package reconcile
 

pkg/deployment/reconcile/action_enable_scaling_cluster.go

@@ -17,6 +17,8 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
+// Author Adam Janikowski
+//
 
 package reconcile
 

pkg/deployment/reconcile/action_maintenance_disable.go

@@ -0,0 +1,72 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/agency"
+	"github.com/rs/zerolog"
+)
+
+func init() {
+	registerAction(api.ActionTypeDisableMaintenance, newDisableMaintenanceAction)
+}
+
+func newDisableMaintenanceAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &actionDisableMaintenance{}
+
+	a.actionImpl = newActionImpl(log, action, actionCtx, addMemberTimeout, &a.newMemberID)
+
+	return a
+}
+
+type actionDisableMaintenance struct {
+	// actionImpl implement timeout and member id functions
+	actionImpl
+
+	actionEmptyCheckProgress
+
+	newMemberID string
+}
+
+func (a *actionDisableMaintenance) Start(ctx context.Context) (bool, error) {
+	switch a.actionCtx.GetMode() {
+	case api.DeploymentModeSingle:
+		return true, nil
+	}
+
+	client, err := a.actionCtx.GetDatabaseClient(ctx)
+	if err != nil {
+		a.log.Error().Err(err).Msgf("Unable to get agency client")
+		return true, nil
+	}
+
+	if err := agency.SetMaintenanceMode(ctx, client, false); err != nil {
+		a.log.Error().Err(err).Msgf("Unable to disable maintenance")
+		return true, nil
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/action_maintenance_enable.go

@@ -0,0 +1,72 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/agency"
+	"github.com/rs/zerolog"
+)
+
+func init() {
+	registerAction(api.ActionTypeEnableMaintenance, newEnableMaintenanceAction)
+}
+
+func newEnableMaintenanceAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &actionEnableMaintenance{}
+
+	a.actionImpl = newActionImpl(log, action, actionCtx, addMemberTimeout, &a.newMemberID)
+
+	return a
+}
+
+type actionEnableMaintenance struct {
+	// actionImpl implement timeout and member id functions
+	actionImpl
+
+	actionEmptyCheckProgress
+
+	newMemberID string
+}
+
+func (a *actionEnableMaintenance) Start(ctx context.Context) (bool, error) {
+	switch a.actionCtx.GetMode() {
+	case api.DeploymentModeSingle:
+		return true, nil
+	}
+
+	client, err := a.actionCtx.GetDatabaseClient(ctx)
+	if err != nil {
+		a.log.Error().Err(err).Msgf("Unable to get agency client")
+		return true, nil
+	}
+
+	if err := agency.SetMaintenanceMode(ctx, client, true); err != nil {
+		a.log.Error().Err(err).Msgf("Unable to set maintenance")
+		return true, nil
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/helper_maintenance.go

@@ -0,0 +1,44 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
+)
+
+func withMaintenance(plan ...api.Action) api.Plan {
+	if !features.Maintenance().Enabled() {
+		return plan
+	}
+
+	var newPlan api.Plan
+
+	newPlan = append(newPlan, api.NewAction(api.ActionTypeEnableMaintenance, api.ServerGroupUnknown, "", "Enable maintenance before actions"))
+
+	newPlan = append(newPlan, plan...)
+
+	newPlan = append(newPlan, api.NewAction(api.ActionTypeDisableMaintenance, api.ServerGroupUnknown, "", "Disable maintenance after actions"))
+
+	return newPlan
+}

pkg/deployment/reconcile/plan_builder.go

@@ -97,13 +97,7 @@ func fetchAgency(ctx context.Context, log zerolog.Logger,
 		agencyCtx, agencyCancel := goContext.WithTimeout(ctx, time.Minute)
 		defer agencyCancel()
 
-		ret := &agency.ArangoPlanDatabases{}
-
-		if err := context.GetAgencyData(agencyCtx, cache, agency.ArangoKey, agency.PlanKey, agency.PlanCollectionsKey); err != nil {
-			return nil, err
-		}
-
-		return ret, nil
+		return agency.GetAgencyCollections(agencyCtx, context.GetAgencyData)
 	} else {
 		return nil, fmt.Errorf("not able to read from agency when agency is down")
 	}
@@ -214,6 +208,10 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
 		plan = pb.Apply(createJWTStatusUpdate)
 	}
 
+	if plan.IsEmpty() {
+		plan = pb.Apply(createMaintenanceManagementPlan)
+	}
+
 	// Check for scale up/down
 	if plan.IsEmpty() {
 		plan = pb.Apply(createScaleMemeberPlan)

pkg/deployment/reconcile/plan_builder_common.go

@@ -0,0 +1,72 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/agency"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	"github.com/rs/zerolog"
+)
+
+func createMaintenanceManagementPlan(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
+	if spec.Mode.Get() == api.DeploymentModeSingle {
+		return nil
+	}
+
+	if !features.Maintenance().Enabled() {
+		// Maintenance feature is not enabled
+		return nil
+	}
+
+	client, err := context.GetDatabaseClient(ctx)
+	if err != nil {
+		log.Error().Err(err).Msgf("Unable to get agency client")
+		return nil
+	}
+
+	m, err := agency.GetMaintenanceMode(ctx, client)
+	if err != nil {
+		log.Error().Err(err).Msgf("Unable to get agency maintenance mode")
+		return nil
+	}
+
+	if !m.Enabled() && spec.Database.GetMaintenance() {
+		log.Info().Msgf("Enabling maintenance mode")
+		return api.Plan{api.NewAction(api.ActionTypeEnableMaintenance, api.ServerGroupUnknown, "")}
+	}
+
+	if m.Enabled() && !spec.Database.GetMaintenance() {
+		log.Info().Msgf("Disabling maintenance mode")
+		return api.Plan{api.NewAction(api.ActionTypeEnableMaintenance, api.ServerGroupUnknown, "")}
+	}
+
+	return nil
+}

pkg/deployment/reconcile/plan_builder_context.go

@@ -25,6 +25,8 @@ package reconcile
 import (
 	"context"
 
+	"github.com/arangodb/go-driver/agency"
+
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
 
 	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
@@ -69,6 +71,8 @@ type PlanBuilderContext interface {
 	GetBackup(backup string) (*backupApi.ArangoBackup, error)
 	// GetName receives deployment name
 	GetName() string
+	// GetAgency returns a connection to the entire agency.
+	GetAgency(ctx context.Context) (agency.Agency, error)
 }
 
 // newPlanBuilderContext creates a PlanBuilderContext from the given context

pkg/deployment/reconcile/plan_builder_restore.go

@@ -75,14 +75,25 @@ func createRestorePlan(ctx context.Context,
 			}
 		}
 
-		return api.Plan{
-			api.NewAction(api.ActionTypeBackupRestore, api.ServerGroupUnknown, ""),
-		}
+		return restorePlan(spec.Mode.Get())
 	}
 
 	return nil
 }
 
+func restorePlan(mode api.DeploymentMode) api.Plan {
+	p := api.Plan{
+		api.NewAction(api.ActionTypeBackupRestore, api.ServerGroupUnknown, ""),
+	}
+
+	switch mode {
+	case api.DeploymentModeActiveFailover:
+		p = withMaintenance(p...)
+	}
+
+	return p
+}
+
 func createRestorePlanEncryption(ctx context.Context, log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus, builderCtx PlanBuilderContext, backup *backupv1.ArangoBackup) (bool, api.Plan) {
 	if spec.RestoreEncryptionSecret != nil {
 		if !spec.RocksDB.IsEncrypted() {

pkg/deployment/resources/license.go

@@ -17,6 +17,8 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
+// Author Adam Janikowski
+//
 
 package resources
 

pkg/deployment/resources/pdbs.go

@@ -17,6 +17,8 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
+// Author Adam Janikowski
+//
 
 package resources