[Feature] Check if the server is cleaned out (#775)

CHANGELOG.md

@@ -5,6 +5,7 @@
 - Add HighPriorityPlan to ArangoDeployment Status
 - Add Pending Member phase
 - Add Ephemeral Volumes for apps feature
+- Check if the DB server is cleaned out.
 
 ## [1.2.1](https://github.com/arangodb/kube-arangodb/tree/1.2.1) (2021-07-28)
 - Fix ArangoMember race with multiple ArangoDeployments within single namespace

pkg/apis/deployment/v1/plan.go

@@ -49,7 +49,7 @@ func (a ActionType) String() string {
 // Priority returns plan priority
 func (a ActionType) Priority() ActionPriority {
 	switch a {
-	case ActionTypeMemberPhaseUpdate, ActionTypeMemberRIDUpdate:
+	case ActionTypeMemberPhaseUpdate, ActionTypeMemberRIDUpdate, ActionTypeSetMemberCondition:
 		return ActionPriorityHigh
 	default:
 		return ActionPriorityNormal
@@ -147,7 +147,7 @@ const (
 	ActionTypeClusterMemberCleanup ActionType = "ClusterMemberCleanup"
 	// ActionTypeEnableMaintenance enables maintenance on cluster.
 	ActionTypeEnableMaintenance ActionType = "EnableMaintenance"
-	// ActionTypeEnableMaintenance disables maintenance on cluster.
+	// ActionTypeDisableMaintenance disables maintenance on cluster.
 	ActionTypeDisableMaintenance ActionType = "DisableMaintenance"
 	// ActionTypeSetMaintenanceCondition sets maintenance condition.
 	ActionTypeSetMaintenanceCondition ActionType = "SetMaintenanceCondition"
@@ -157,6 +157,8 @@ const (
 	ActionTypeBootstrapSetPassword ActionType = "BootstrapSetPassword"
 	// ActionTypeMemberPhaseUpdate updated member phase. High priority
 	ActionTypeMemberPhaseUpdate ActionType = "MemberPhaseUpdate"
+	// ActionTypeSetMemberCondition sets member condition. It is high priority action.
+	ActionTypeSetMemberCondition ActionType = "SetMemberCondition"
 	// ActionTypeMemberRIDUpdate updated member Run ID (UID). High priority
 	ActionTypeMemberRIDUpdate ActionType = "MemberRIDUpdate"
 )

pkg/apis/deployment/v2alpha1/plan.go

@@ -49,7 +49,7 @@ func (a ActionType) String() string {
 // Priority returns plan priority
 func (a ActionType) Priority() ActionPriority {
 	switch a {
-	case ActionTypeMemberPhaseUpdate, ActionTypeMemberRIDUpdate:
+	case ActionTypeMemberPhaseUpdate, ActionTypeMemberRIDUpdate, ActionTypeSetMemberCondition:
 		return ActionPriorityHigh
 	default:
 		return ActionPriorityNormal
@@ -147,7 +147,7 @@ const (
 	ActionTypeClusterMemberCleanup ActionType = "ClusterMemberCleanup"
 	// ActionTypeEnableMaintenance enables maintenance on cluster.
 	ActionTypeEnableMaintenance ActionType = "EnableMaintenance"
-	// ActionTypeEnableMaintenance disables maintenance on cluster.
+	// ActionTypeDisableMaintenance disables maintenance on cluster.
 	ActionTypeDisableMaintenance ActionType = "DisableMaintenance"
 	// ActionTypeSetMaintenanceCondition sets maintenance condition.
 	ActionTypeSetMaintenanceCondition ActionType = "SetMaintenanceCondition"
@@ -157,6 +157,8 @@ const (
 	ActionTypeBootstrapSetPassword ActionType = "BootstrapSetPassword"
 	// ActionTypeMemberPhaseUpdate updated member phase. High priority
 	ActionTypeMemberPhaseUpdate ActionType = "MemberPhaseUpdate"
+	// ActionTypeSetMemberCondition sets member condition. It is high priority action.
+	ActionTypeSetMemberCondition ActionType = "SetMemberCondition"
 	// ActionTypeMemberRIDUpdate updated member Run ID (UID). High priority
 	ActionTypeMemberRIDUpdate ActionType = "MemberRIDUpdate"
 )

pkg/deployment/reconcile/action_cleanout_member.go

@@ -178,7 +178,7 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, e
 	}
 	// Cleanout completed
 	if m.Conditions.Update(api.ConditionTypeCleanedOut, true, "CleanedOut", "") {
-		if a.actionCtx.UpdateMember(ctx, m); err != nil {
+		if err := a.actionCtx.UpdateMember(ctx, m); err != nil {
 			return false, false, errors.WithStack(err)
 		}
 	}

pkg/deployment/reconcile/action_set_member_condition.go

@@ -0,0 +1,84 @@
+//
+// DISCLAIMER
+//
+// Copyright 2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Tomasz Mielech
+//
+
+package reconcile
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/rs/zerolog"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
+)
+
+func init() {
+	registerAction(api.ActionTypeSetMemberCondition, setMemberCondition)
+}
+
+func setMemberCondition(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &actionSetMemberCondition{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
+
+	return a
+}
+
+type actionSetMemberCondition struct {
+	// actionImpl implement timeout and member id functions
+	actionImpl
+
+	actionEmptyCheckProgress
+}
+
+// Start starts the action for changing conditions on the provided member.
+func (a actionSetMemberCondition) Start(ctx context.Context) (bool, error) {
+	m, ok := a.actionCtx.GetMemberStatusByID(a.action.MemberID)
+	if !ok {
+		a.log.Info().Msg("can not set the condition because the member is gone already")
+		return true, nil
+	}
+
+	if len(a.action.Params) == 0 {
+		a.log.Info().Msg("can not start the action with the empty list of conditions")
+		return true, nil
+	}
+
+	for condition, value := range a.action.Params {
+		set, err := strconv.ParseBool(value)
+		if err != nil {
+			a.log.Error().Err(err).Str("value", value).Msg("can not parse string to boolean")
+			continue
+		}
+
+		a.log.Debug().Msg("set the condition")
+
+		m.Conditions.Update(api.ConditionType(condition), set, a.action.Reason, "action set the member condition")
+	}
+
+	if err := a.actionCtx.UpdateMember(ctx, m); err != nil {
+		return false, errors.Wrap(errors.WithStack(err), "can not update the member")
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/plan_builder_clean_out.go

@@ -0,0 +1,91 @@
+//
+// DISCLAIMER
+//
+// Copyright 2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Tomasz Mielech
+//
+
+package reconcile
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/arangodb/go-driver"
+	"github.com/rs/zerolog"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util/arangod"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector"
+)
+
+// createCleanOutPlan creates clean out action if the server is cleaned out and the operator is not aware of it.
+func createCleanOutPlan(ctx context.Context, log zerolog.Logger, _ k8sutil.APIObject, spec api.DeploymentSpec,
+	status api.DeploymentStatus, _ inspectorInterface.Inspector, planCtx PlanBuilderContext) api.Plan {
+
+	if spec.GetMode() != api.DeploymentModeCluster {
+		return nil
+	}
+
+	cluster, err := getCluster(ctx, planCtx)
+	if err != nil {
+		log.Warn().Err(err).Msgf("Unable to get cluster")
+		return nil
+	}
+
+	ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout())
+	defer cancel()
+	health, err := cluster.Health(ctxChild)
+	if err != nil {
+		log.Warn().Err(err).Msgf("Unable to get cluster health")
+		return nil
+	}
+
+	for id, member := range health.Health {
+		switch member.Role {
+		case driver.ServerRoleDBServer:
+			memberStatus, ok := status.Members.DBServers.ElementByID(string(id))
+			if !ok {
+				continue
+			}
+
+			if memberStatus.Conditions.IsTrue(api.ConditionTypeCleanedOut) {
+				continue
+			}
+
+			if isCleanedOut, err := cluster.IsCleanedOut(ctx, string(id)); err != nil {
+				log.Warn().Err(err).Str("id", string(id)).Msgf("Unable to get clean out status")
+				return nil
+			} else if isCleanedOut {
+				log.Info().
+					Str("role", string(member.Role)).
+					Str("id", string(id)).
+					Msgf("server is cleaned out so operator must do the same")
+
+				action := api.NewAction(api.ActionTypeSetMemberCondition, api.ServerGroupDBServers, string(id),
+					"server is cleaned out so operator must do the same")
+				action = action.AddParam(string(api.ConditionTypeCleanedOut), strconv.FormatBool(true))
+
+				return api.Plan{action}
+			}
+		}
+	}
+
+	return nil
+}

pkg/deployment/reconcile/plan_builder_high.go

@@ -86,6 +86,10 @@ func createHighPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.A
 		plan = pb.Apply(updateMemberPhasePlan)
 	}
 
+	if plan.IsEmpty() {
+		plan = pb.Apply(createCleanOutPlan)
+	}
+
 	// Return plan
 	return plan, true
 }

pkg/deployment/reconcile/utils.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,15 +18,21 @@
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
 // Author Adam Janikowski
+// Author Tomasz Mielech
 //
 
 package reconcile
 
 import (
+	"context"
 	"sort"
 
-	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/arangodb/go-driver"
 	core "k8s.io/api/core/v1"
+
+	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/arangodb/kube-arangodb/pkg/util/arangod"
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 )
 
 func secretKeysToListWithPrefix(s *core.Secret) []string {
@@ -44,3 +50,22 @@ func secretKeysToList(s *core.Secret) []string {
 
 	return keys
 }
+
+// getCluster returns the cluster connection.
+func getCluster(ctx context.Context, planCtx PlanBuilderContext) (driver.Cluster, error) {
+	ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout())
+	defer cancel()
+	c, err := planCtx.GetDatabaseClient(ctxChild)
+	if err != nil {
+		return nil, errors.WithStack(errors.Wrapf(err, "Unable to get database client"))
+	}
+
+	ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
+	defer cancel()
+	cluster, err := c.Cluster(ctxChild)
+	if err != nil {
+		return nil, errors.WithStack(errors.Wrapf(err, "Unable to get cluster client"))
+	}
+
+	return cluster, nil
+}