kube-arangodb/pkg/deployment/reconcile/plan_builder_member_recovery.go

//
// DISCLAIMER
//
// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//

package reconcile

import (
	"context"

	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
	"github.com/arangodb/kube-arangodb/pkg/deployment/actions"
	"github.com/arangodb/kube-arangodb/pkg/deployment/agency/state"
	sharedReconcile "github.com/arangodb/kube-arangodb/pkg/deployment/reconcile/shared"
	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
)

// createMemberFailedRestoreNormalPlan returns only actions which are not recreate member.
func (r *Reconciler) createMemberFailedRestoreNormalPlan(ctx context.Context, apiObject k8sutil.APIObject,
	spec api.DeploymentSpec, status api.DeploymentStatus, context PlanBuilderContext) api.Plan {
	condition := func(a api.Action) bool {
		return a.Type != api.ActionTypeRecreateMember
	}

	return r.createMemberFailedRestoreInternal(ctx, apiObject, spec, status, context).Filter(condition)
}

// createMemberFailedRestoreHighPlan returns only recreate member actions.
func (r *Reconciler) createMemberFailedRestoreHighPlan(ctx context.Context, apiObject k8sutil.APIObject,
	spec api.DeploymentSpec, status api.DeploymentStatus, context PlanBuilderContext) api.Plan {
	condition := func(a api.Action) bool {
		return a.Type == api.ActionTypeRecreateMember
	}

	return r.createMemberFailedRestoreInternal(ctx, apiObject, spec, status, context).Filter(condition)
}

func (r *Reconciler) createMemberFailedRestoreInternal(_ context.Context, _ k8sutil.APIObject, spec api.DeploymentSpec,
	status api.DeploymentStatus, context PlanBuilderContext) api.Plan {
	var plan api.Plan

	// Fetch agency plan.
	agencyState, agencyOK := context.GetAgencyCache()

	// Check for members in failed state.
	for _, group := range api.AllServerGroups {
		members := status.Members.MembersOfGroup(group)
		failed := 0
		for _, m := range members {
			if m.Phase == api.MemberPhaseFailed {
				failed++
			}
		}

		for _, m := range members {
			if m.Phase != api.MemberPhaseFailed || len(plan) > 0 {
				continue
			}

			memberLog := r.log.Str("id", m.ID).Str("role", group.AsRole())

			if group == api.ServerGroupDBServers && spec.GetMode() == api.DeploymentModeCluster {
				if !agencyOK {
					// If agency is down DBServers should not be touched.
					memberLog.Info("Agency state is not present")
					continue
				}

				if agencyState.Target.CleanedServers.Contains(state.Server(m.ID)) {
					memberLog.Info("Member is CleanedOut")
					continue
				}

				if agencyState.Plan.Collections.IsDBServerLeader(state.Server(m.ID)) {
					memberLog.Info("Recreating leader DBServer - it cannot be removed gracefully")
					plan = append(plan, actions.NewAction(api.ActionTypeRecreateMember, group, m))

					continue
				}

				if c := spec.DBServers.GetCount(); c <= len(members)-failed {
					// There are more or equal alive members than current count. A member should not be recreated.
					continue
				}

				if agencyState.Plan.Collections.IsDBServerPresent(state.Server(m.ID)) {
					// DBServer still exists in agency plan! Will not be removed, but needs to be recreated.
					memberLog.Info("Recreating DBServer - it cannot be removed gracefully")
					plan = append(plan, actions.NewAction(api.ActionTypeRecreateMember, group, m))

					continue
				}
				// From here on, DBServer can be recreated.
			}

			switch group {
			case api.ServerGroupAgents:
				// For agents just recreate member do not rotate ID, do not remove PVC or service.
				memberLog.Info("Restoring old member. For agency members recreation of PVC is not supported - to prevent DataLoss")
				plan = append(plan, actions.NewAction(api.ActionTypeRecreateMember, group, m))
			case api.ServerGroupSingle:
				// Do not remove data for single.
				memberLog.Info("Restoring old member. Rotation for single servers is not safe")
				plan = append(plan, actions.NewAction(api.ActionTypeRecreateMember, group, m))
			default:
				if spec.GetAllowMemberRecreation(group) {
					memberLog.Info("Creating member replacement plan because member has failed")
					plan = append(plan,
						actions.NewAction(api.ActionTypeRemoveMember, group, m),
						actions.NewAction(api.ActionTypeAddMember, group, sharedReconcile.WithPredefinedMember("")),
						actions.NewAction(api.ActionTypeWaitForMemberUp, group, sharedReconcile.WithPredefinedMember(api.MemberIDPreviousAction)),
					)
				} else {
					memberLog.Info("Restoring old member. Recreation is disabled for group")
					plan = append(plan, actions.NewAction(api.ActionTypeRecreateMember, group, m))
				}
			}
		}
	}

	if len(plan) == 0 && !agencyOK {
		r.log.Warn("unable to build further plan without access to agency")
		plan = append(plan, actions.NewClusterAction(api.ActionTypeIdle))
	}

	return plan
}