[Feature] CreationFailed ArangoMember Phase (#1237)

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
 - (Feature) Add Generics & Drop policy/v1beta1 support
 - (Feature) Add Kubernetes Client logger
+- (Feature) CreationFailed ArangoMember Phase
 
 ## [1.2.24](https://github.com/arangodb/kube-arangodb/tree/1.2.24) (2023-01-25)
 - (Bugfix) Fix deployment creation on ARM64

Makefile

@@ -519,10 +519,10 @@ vendor:
 	@go mod vendor -e
 
 set-deployment-api-version-v2alpha1: export API_VERSION=2alpha1
-set-deployment-api-version-v2alpha1: set-api-version/deployment set-typed-api-version/deployment set-api-version/replication
+set-deployment-api-version-v2alpha1: set-api-version/deployment set-api-version/replication
 
 set-deployment-api-version-v1: export API_VERSION=1
-set-deployment-api-version-v1: set-api-version/deployment set-typed-api-version/deployment set-api-version/replication
+set-deployment-api-version-v1: set-api-version/deployment set-api-version/replication
 
 set-typed-api-version/%:
 	@grep -rHn "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/typed/$*/v[A-Za-z0-9]\+" \

pkg/apis/deployment/v1/member_phase.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -30,6 +30,8 @@ const (
 	MemberPhasePending MemberPhase = "Pending"
 	// MemberPhaseCreated indicates that all resources needed for the member have been created
 	MemberPhaseCreated MemberPhase = "Created"
+	// MemberPhaseCreationFailed indicates that creation of member resources was not possible, fallback to MemberPhaseCreated state
+	MemberPhaseCreationFailed MemberPhase = "CreationFailed"
 	// MemberPhaseFailed indicates that the member is gone beyond hope of recovery. It must be replaced with a new member.
 	MemberPhaseFailed MemberPhase = "Failed"
 	// MemberPhaseCleanOut indicates that a dbserver is in the process of being cleaned out
@@ -50,7 +52,7 @@ const (
 
 // IsPending returns true when given phase == "" OR "Pending"
 func (p MemberPhase) IsPending() bool {
-	return p == MemberPhaseNone || p == MemberPhasePending
+	return p == MemberPhaseNone || p == MemberPhasePending || p == MemberPhaseCreationFailed
 }
 
 // IsFailed returns true when given phase == "Failed"
@@ -76,7 +78,7 @@ func (p MemberPhase) String() string {
 // GetPhase parses string into phase
 func GetPhase(phase string) (MemberPhase, bool) {
 	switch p := MemberPhase(phase); p {
-	case MemberPhaseNone, MemberPhasePending, MemberPhaseCreated, MemberPhaseFailed, MemberPhaseCleanOut, MemberPhaseDrain, MemberPhaseResign, MemberPhaseShuttingDown, MemberPhaseRotating, MemberPhaseRotateStart, MemberPhaseUpgrading:
+	case MemberPhaseNone, MemberPhasePending, MemberPhaseCreated, MemberPhaseCreationFailed, MemberPhaseFailed, MemberPhaseCleanOut, MemberPhaseDrain, MemberPhaseResign, MemberPhaseShuttingDown, MemberPhaseRotating, MemberPhaseRotateStart, MemberPhaseUpgrading:
 		return p, true
 	default:
 		return "", false

pkg/apis/deployment/v2alpha1/member_phase.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -30,6 +30,8 @@ const (
 	MemberPhasePending MemberPhase = "Pending"
 	// MemberPhaseCreated indicates that all resources needed for the member have been created
 	MemberPhaseCreated MemberPhase = "Created"
+	// MemberPhaseCreationFailed indicates that creation of member resources was not possible, fallback to MemberPhaseCreated state
+	MemberPhaseCreationFailed MemberPhase = "CreationFailed"
 	// MemberPhaseFailed indicates that the member is gone beyond hope of recovery. It must be replaced with a new member.
 	MemberPhaseFailed MemberPhase = "Failed"
 	// MemberPhaseCleanOut indicates that a dbserver is in the process of being cleaned out
@@ -50,7 +52,7 @@ const (
 
 // IsPending returns true when given phase == "" OR "Pending"
 func (p MemberPhase) IsPending() bool {
-	return p == MemberPhaseNone || p == MemberPhasePending
+	return p == MemberPhaseNone || p == MemberPhasePending || p == MemberPhaseCreationFailed
 }
 
 // IsFailed returns true when given phase == "Failed"
@@ -76,7 +78,7 @@ func (p MemberPhase) String() string {
 // GetPhase parses string into phase
 func GetPhase(phase string) (MemberPhase, bool) {
 	switch p := MemberPhase(phase); p {
-	case MemberPhaseNone, MemberPhasePending, MemberPhaseCreated, MemberPhaseFailed, MemberPhaseCleanOut, MemberPhaseDrain, MemberPhaseResign, MemberPhaseShuttingDown, MemberPhaseRotating, MemberPhaseRotateStart, MemberPhaseUpgrading:
+	case MemberPhaseNone, MemberPhasePending, MemberPhaseCreated, MemberPhaseCreationFailed, MemberPhaseFailed, MemberPhaseCleanOut, MemberPhaseDrain, MemberPhaseResign, MemberPhaseShuttingDown, MemberPhaseRotating, MemberPhaseRotateStart, MemberPhaseUpgrading:
 		return p, true
 	default:
 		return "", false

pkg/deployment/reconcile/plan_builder_high.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -105,6 +105,15 @@ func (r *Reconciler) updateMemberPhasePlan(ctx context.Context, apiObject k8suti
 			)
 			plan = append(plan, p...)
 		}
+
+		if e.Member.Phase == api.MemberPhaseCreationFailed {
+			var p api.Plan
+			p = append(p,
+				actions.NewAction(api.ActionTypeMemberPhaseUpdate, e.Group, e.Member,
+					"Move to None phase due to Creation Error").AddParam(actionTypeMemberPhaseUpdatePhaseKey, api.MemberPhaseNone.String()),
+			)
+			plan = append(plan, p...)
+		}
 	}
 
 	return plan

pkg/deployment/resilience/member_failure.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -50,7 +50,7 @@ func (r *Resilience) CheckMemberFailure(ctx context.Context) error {
 
 		// Check if there are Members with Phase Upgrading or Rotation but no plan
 		switch m.Phase {
-		case api.MemberPhaseNone, api.MemberPhasePending:
+		case api.MemberPhaseNone, api.MemberPhasePending, api.MemberPhaseCreationFailed:
 			continue
 		case api.MemberPhaseUpgrading, api.MemberPhaseRotating, api.MemberPhaseCleanOut, api.MemberPhaseRotateStart, api.MemberPhaseShuttingDown:
 			if len(status.Plan) == 0 {

pkg/deployment/resources/pod_creator.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -40,6 +40,7 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/member"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/reconciler"
 	"github.com/arangodb/kube-arangodb/pkg/util"
 	"github.com/arangodb/kube-arangodb/pkg/util/constants"
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
@@ -438,8 +439,12 @@ func (r *Resources) createPodForMember(ctx context.Context, cachedStatus inspect
 
 		ctxChild, cancel := globals.GetGlobalTimeouts().Kubernetes().WithTimeout(ctx)
 		defer cancel()
+
 		podName, uid, err := CreateArangoPod(ctxChild, cachedStatus.PodsModInterface().V1(), apiObject, spec, group, CreatePodFromTemplate(template.PodSpec))
 		if err != nil {
+			if uerr := r.context.WithMemberStatusUpdateErr(ctx, m.ID, group, updateMemberPhase(api.MemberPhaseCreationFailed)); uerr != nil {
+				return errors.WithStack(uerr)
+			}
 			return errors.WithStack(err)
 		}
 
@@ -486,6 +491,9 @@ func (r *Resources) createPodForMember(ctx context.Context, cachedStatus inspect
 		defer cancel()
 		podName, uid, err := CreateArangoPod(ctxChild, cachedStatus.PodsModInterface().V1(), apiObject, spec, group, CreatePodFromTemplate(template.PodSpec))
 		if err != nil {
+			if uerr := r.context.WithMemberStatusUpdateErr(ctx, m.ID, group, updateMemberPhase(api.MemberPhaseCreationFailed)); uerr != nil {
+				return errors.WithStack(uerr)
+			}
 			return errors.WithStack(err)
 		}
 
@@ -691,3 +699,19 @@ func CreatePodSuffix(spec api.DeploymentSpec) string {
 	hash := sha1.Sum(raw)
 	return fmt.Sprintf("%0x", hash)[:6]
 }
+
+func updateMemberPhase(phase api.MemberPhase) reconciler.DeploymentMemberStatusUpdateErrFunc {
+	return func(s *api.MemberStatus) (bool, error) {
+		if s == nil {
+			return false, nil
+		}
+
+		if s.Phase == phase {
+			return false, nil
+		}
+
+		s.Phase = phase
+
+		return true, nil
+	}
+}

pkg/deployment/resources/pod_inspector.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -471,7 +471,7 @@ func (r *Resources) InspectPods(ctx context.Context, cachedStatus inspectorInter
 			if _, exists := cachedStatus.Pod().V1().GetSimple(podName); !exists {
 				log.Str("pod-name", podName).Debug("Does not exist")
 				switch m.Phase {
-				case api.MemberPhaseNone, api.MemberPhasePending:
+				case api.MemberPhaseNone, api.MemberPhasePending, api.MemberPhaseCreationFailed:
 					// Do nothing
 					log.Str("pod-name", podName).Debug("PodPhase is None, waiting for the pod to be recreated")
 				case api.MemberPhaseShuttingDown, api.MemberPhaseUpgrading, api.MemberPhaseFailed, api.MemberPhaseRotateStart, api.MemberPhaseRotating: