[Feature] Agent recreation and agency info (#790)

CHANGELOG.md

@@ -4,6 +4,7 @@
 - Update UBI Image to 8.4
 - Fix ArangoSync Liveness Prove
 - Allow runtime update of Sidecar images
+- Allow Agent recreation with preserved IDs
 
 ## [1.2.2](https://github.com/arangodb/kube-arangodb/tree/1.2.2) (2021-09-09)
 - Update 'github.com/arangodb/arangosync-client' dependency to v0.7.0

pkg/apis/deployment/v1/deployment_status.go

@@ -78,6 +78,9 @@ type DeploymentStatus struct {
 
 	// ForceStatusReload if set to true forces a reload of the status from the custom resource.
 	ForceStatusReload *bool `json:"force-status-reload,omitempty"`
+
+	// Agency keeps information about agency
+	Agency *DeploymentStatusAgencyInfo `json:"agency,omitempty"`
 }
 
 // Equal checks for equality
@@ -95,7 +98,8 @@ func (ds *DeploymentStatus) Equal(other DeploymentStatus) bool {
 		ds.Conditions.Equal(other.Conditions) &&
 		ds.Plan.Equal(other.Plan) &&
 		ds.AcceptedSpec.Equal(other.AcceptedSpec) &&
-		ds.SecretHashes.Equal(other.SecretHashes)
+		ds.SecretHashes.Equal(other.SecretHashes) &&
+		ds.Agency.Equal(other.Agency)
 }
 
 // IsForceReload returns true if ForceStatusReload is set to true

pkg/apis/deployment/v1/deployment_status_agency_info.go

@@ -0,0 +1,79 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+import (
+	"sort"
+	"strings"
+)
+
+type DeploymentStatusAgencySize int
+
+func (d *DeploymentStatusAgencySize) Equal(b *DeploymentStatusAgencySize) bool {
+	if d == nil && b == nil {
+		return true
+	}
+
+	if d == nil || b == nil {
+		return true
+	}
+
+	return *d == *b
+}
+
+type DeploymentStatusAgencyIDs []string
+
+func (d DeploymentStatusAgencyIDs) Sort() {
+	sort.Slice(d, func(i, j int) bool {
+		return strings.Compare(d[i], d[j]) > 0
+	})
+}
+
+func (d DeploymentStatusAgencyIDs) Equal(b DeploymentStatusAgencyIDs) bool {
+	if len(d) != len(b) {
+		return false
+	}
+
+	for id := range d {
+		if d[id] != b[id] {
+			return false
+		}
+	}
+
+	return true
+}
+
+type DeploymentStatusAgencyInfo struct {
+	Size *DeploymentStatusAgencySize `json:"size,omitempty"`
+	IDs  DeploymentStatusAgencyIDs   `json:"ids,omitempty"`
+}
+
+func (d *DeploymentStatusAgencyInfo) Equal(b *DeploymentStatusAgencyInfo) bool {
+	if d == nil && b == nil {
+		return true
+	}
+
+	if d == nil || b == nil {
+		return true
+	}
+
+	return d.IDs.Equal(b.IDs) && d.Size.Equal(b.Size)
+}

pkg/apis/deployment/v1/zz_generated.deepcopy.go

@@ -679,6 +679,11 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.Agency != nil {
+		in, out := &in.Agency, &out.Agency
+		*out = new(DeploymentStatusAgencyInfo)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -692,6 +697,52 @@ func (in *DeploymentStatus) DeepCopy() *DeploymentStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in DeploymentStatusAgencyIDs) DeepCopyInto(out *DeploymentStatusAgencyIDs) {
+	{
+		in := &in
+		*out = make(DeploymentStatusAgencyIDs, len(*in))
+		copy(*out, *in)
+		return
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusAgencyIDs.
+func (in DeploymentStatusAgencyIDs) DeepCopy() DeploymentStatusAgencyIDs {
+	if in == nil {
+		return nil
+	}
+	out := new(DeploymentStatusAgencyIDs)
+	in.DeepCopyInto(out)
+	return *out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DeploymentStatusAgencyInfo) DeepCopyInto(out *DeploymentStatusAgencyInfo) {
+	*out = *in
+	if in.Size != nil {
+		in, out := &in.Size, &out.Size
+		*out = new(DeploymentStatusAgencySize)
+		**out = **in
+	}
+	if in.IDs != nil {
+		in, out := &in.IDs, &out.IDs
+		*out = make(DeploymentStatusAgencyIDs, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusAgencyInfo.
+func (in *DeploymentStatusAgencyInfo) DeepCopy() *DeploymentStatusAgencyInfo {
+	if in == nil {
+		return nil
+	}
+	out := new(DeploymentStatusAgencyInfo)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentStatusHashes) DeepCopyInto(out *DeploymentStatusHashes) {
 	*out = *in

pkg/apis/deployment/v2alpha1/deployment_status.go

@@ -78,6 +78,9 @@ type DeploymentStatus struct {
 
 	// ForceStatusReload if set to true forces a reload of the status from the custom resource.
 	ForceStatusReload *bool `json:"force-status-reload,omitempty"`
+
+	// Agency keeps information about agency
+	Agency *DeploymentStatusAgencyInfo `json:"agency,omitempty"`
 }
 
 // Equal checks for equality
@@ -95,7 +98,8 @@ func (ds *DeploymentStatus) Equal(other DeploymentStatus) bool {
 		ds.Conditions.Equal(other.Conditions) &&
 		ds.Plan.Equal(other.Plan) &&
 		ds.AcceptedSpec.Equal(other.AcceptedSpec) &&
-		ds.SecretHashes.Equal(other.SecretHashes)
+		ds.SecretHashes.Equal(other.SecretHashes) &&
+		ds.Agency.Equal(other.Agency)
 }
 
 // IsForceReload returns true if ForceStatusReload is set to true

pkg/apis/deployment/v2alpha1/deployment_status_agency_info.go

@@ -0,0 +1,79 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v2alpha1
+
+import (
+	"sort"
+	"strings"
+)
+
+type DeploymentStatusAgencySize int
+
+func (d *DeploymentStatusAgencySize) Equal(b *DeploymentStatusAgencySize) bool {
+	if d == nil && b == nil {
+		return true
+	}
+
+	if d == nil || b == nil {
+		return true
+	}
+
+	return *d == *b
+}
+
+type DeploymentStatusAgencyIDs []string
+
+func (d DeploymentStatusAgencyIDs) Sort() {
+	sort.Slice(d, func(i, j int) bool {
+		return strings.Compare(d[i], d[j]) > 0
+	})
+}
+
+func (d DeploymentStatusAgencyIDs) Equal(b DeploymentStatusAgencyIDs) bool {
+	if len(d) != len(b) {
+		return false
+	}
+
+	for id := range d {
+		if d[id] != b[id] {
+			return false
+		}
+	}
+
+	return true
+}
+
+type DeploymentStatusAgencyInfo struct {
+	Size *DeploymentStatusAgencySize `json:"size,omitempty"`
+	IDs  DeploymentStatusAgencyIDs   `json:"ids,omitempty"`
+}
+
+func (d *DeploymentStatusAgencyInfo) Equal(b *DeploymentStatusAgencyInfo) bool {
+	if d == nil && b == nil {
+		return true
+	}
+
+	if d == nil || b == nil {
+		return true
+	}
+
+	return d.IDs.Equal(b.IDs) && d.Size.Equal(b.Size)
+}

pkg/apis/deployment/v2alpha1/plan.go

@@ -49,7 +49,7 @@ func (a ActionType) String() string {
 // Priority returns plan priority
 func (a ActionType) Priority() ActionPriority {
 	switch a {
-	case ActionTypeMemberPhaseUpdate, ActionTypeMemberRIDUpdate, ActionTypeSetMemberCondition:
+	case ActionTypeMemberPhaseUpdate, ActionTypeMemberRIDUpdate, ActionTypeSetMemberCondition, ActionTypeBootstrapSetAgencyInfo:
 		return ActionPriorityHigh
 	default:
 		return ActionPriorityNormal
@@ -165,6 +165,8 @@ const (
 	ActionTypeArangoMemberUpdatePodSpec ActionType = "ArangoMemberUpdatePodSpec"
 	// ActionTypeArangoMemberUpdatePodStatus updates pod spec
 	ActionTypeArangoMemberUpdatePodStatus ActionType = "ArangoMemberUpdatePodStatus"
+	// ActionTypeBootstrapSetAgencyInfo set agency info into state
+	ActionTypeBootstrapSetAgencyInfo ActionType = "BootstrapSetAgencyInfo"
 
 	// Runtime Updates
 	// ActionTypeRuntimeContainerImageUpdate updates container image in runtime

pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go

@@ -679,6 +679,11 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.Agency != nil {
+		in, out := &in.Agency, &out.Agency
+		*out = new(DeploymentStatusAgencyInfo)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -692,6 +697,52 @@ func (in *DeploymentStatus) DeepCopy() *DeploymentStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in DeploymentStatusAgencyIDs) DeepCopyInto(out *DeploymentStatusAgencyIDs) {
+	{
+		in := &in
+		*out = make(DeploymentStatusAgencyIDs, len(*in))
+		copy(*out, *in)
+		return
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusAgencyIDs.
+func (in DeploymentStatusAgencyIDs) DeepCopy() DeploymentStatusAgencyIDs {
+	if in == nil {
+		return nil
+	}
+	out := new(DeploymentStatusAgencyIDs)
+	in.DeepCopyInto(out)
+	return *out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DeploymentStatusAgencyInfo) DeepCopyInto(out *DeploymentStatusAgencyInfo) {
+	*out = *in
+	if in.Size != nil {
+		in, out := &in.Size, &out.Size
+		*out = new(DeploymentStatusAgencySize)
+		**out = **in
+	}
+	if in.IDs != nil {
+		in, out := &in.IDs, &out.IDs
+		*out = make(DeploymentStatusAgencyIDs, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusAgencyInfo.
+func (in *DeploymentStatusAgencyInfo) DeepCopy() *DeploymentStatusAgencyInfo {
+	if in == nil {
+		return nil
+	}
+	out := new(DeploymentStatusAgencyInfo)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentStatusHashes) DeepCopyInto(out *DeploymentStatusHashes) {
 	*out = *in

pkg/deployment/deployment.go

@@ -262,6 +262,11 @@ func (d *Deployment) send(ev *deploymentEvent) {
 func (d *Deployment) run() {
 	log := d.deps.Log
 
+	// Create agency mapping
+	if err := d.createAgencyMapping(context.TODO()); err != nil {
+		d.CreateEvent(k8sutil.NewErrorEvent("Failed to create agency mapping members", err, d.GetAPIObject()))
+	}
+
 	if d.GetPhase() == api.DeploymentPhaseNone {
 		// Create service monitor
 		if d.haveServiceMonitorCRD {

pkg/deployment/members.go

@@ -24,11 +24,11 @@ package deployment
 
 import (
 	"context"
-	"strings"
+
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/names"
 
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 
-	"github.com/dchest/uniuri"
 	"github.com/rs/zerolog"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -71,19 +71,78 @@ func (d *Deployment) createInitialMembers(ctx context.Context, apiObject *api.Ar
 	return nil
 }
 
+func (d *Deployment) createAgencyMapping(ctx context.Context) error {
+	spec := d.GetSpec()
+	status, _ := d.GetStatus()
+
+	if !spec.Mode.HasAgents() {
+		return nil
+	}
+
+	if status.Agency != nil {
+		return nil
+	}
+
+	var i api.DeploymentStatusAgencyInfo
+
+	if spec.Agents.Count == nil {
+		return nil
+	}
+
+	agents := status.Members.Agents
+
+	if len(agents) > *spec.Agents.Count {
+		return errors.Newf("Agency size is bigger than requested size")
+	}
+
+	c := api.DeploymentStatusAgencySize(*spec.Agents.Count)
+
+	i.Size = &c
+
+	for id := range agents {
+		i.IDs = append(i.IDs, agents[id].ID)
+	}
+
+	for len(i.IDs) < *spec.Agents.Count {
+		i.IDs = append(i.IDs, names.GetArangodID(api.ServerGroupAgents))
+	}
+
+	return d.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool {
+		s.Agency = &i
+		return true
+	})
+}
+
 // createMember creates member and adds it to the applicable member list.
 // Note: This does not create any pods of PVCs
 // Note: The updated status is not yet written to the apiserver.
 func createMember(log zerolog.Logger, status *api.DeploymentStatus, group api.ServerGroup, id string, apiObject *api.ArangoDeployment) (string, error) {
-	if id == "" {
-		idPrefix := getArangodIDPrefix(group)
-		for {
-			id = idPrefix + strings.ToLower(uniuri.NewLen(8)) // K8s accepts only lowercase, so we use it here as well
-			if !status.Members.ContainsID(id) {
-				break
-			}
-			// Duplicate, try again
+	if group == api.ServerGroupAgents {
+		if status.Agency == nil {
+			return "", errors.New("Agency is not yet defined")
 		}
+		// In case of agents we need to use hardcoded ids
+		if id == "" {
+			for _, nid := range status.Agency.IDs {
+				if !status.Members.ContainsID(nid) {
+					id = nid
+					break
+				}
+			}
+		}
+	} else {
+		if id == "" {
+			for {
+				id = names.GetArangodID(group)
+				if !status.Members.ContainsID(id) {
+					break
+				}
+				// Duplicate, try again
+			}
+		}
+	}
+	if id == "" {
+		return "nil", errors.New("Unable to get ID")
 	}
 	deploymentName := apiObject.GetName()
 	role := group.AsRole()
@@ -167,20 +226,3 @@ func createMember(log zerolog.Logger, status *api.DeploymentStatus, group api.Se
 
 	return id, nil
 }
-
-// getArangodIDPrefix returns the prefix required ID's of arangod servers
-// in the given group.
-func getArangodIDPrefix(group api.ServerGroup) string {
-	switch group {
-	case api.ServerGroupSingle:
-		return "SNGL-"
-	case api.ServerGroupCoordinators:
-		return "CRDN-"
-	case api.ServerGroupDBServers:
-		return "PRMR-"
-	case api.ServerGroupAgents:
-		return "AGNT-"
-	default:
-		return ""
-	}
-}

pkg/deployment/reconcile/action_add_member.go

@@ -85,7 +85,7 @@ func (a *actionAddMember) ActionPlanAppender(current api.Plan) (api.Plan, bool)
 		app = append(app, api.NewAction(api.ActionTypeWaitForMemberUp, a.action.Group, a.newMemberID, "Wait for member in sync after creation"))
 	}
 
-	if _, ok := a.action.Params[api.ActionTypeWaitForMemberUp.String()]; ok {
+	if _, ok := a.action.Params[api.ActionTypeWaitForMemberInSync.String()]; ok {
 		app = append(app, api.NewAction(api.ActionTypeWaitForMemberInSync, a.action.Group, a.newMemberID, "Wait for member in sync after creation"))
 	}
 

pkg/deployment/reconcile/action_mark_to_remove_member.go

@@ -50,7 +50,7 @@ type actionMarkToRemove struct {
 }
 
 func (a *actionMarkToRemove) Start(ctx context.Context) (bool, error) {
-	if a.action.Group != api.ServerGroupDBServers {
+	if a.action.Group != api.ServerGroupDBServers && a.action.Group != api.ServerGroupAgents {
 		return true, nil
 	}
 

pkg/deployment/reconcile/plan_builder_rotate_upgrade.go

@@ -188,9 +188,11 @@ func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.API
 			}
 
 			if pod.Annotations != nil {
-				if _, ok := pod.Annotations[deployment.ArangoDeploymentPodReplaceAnnotation]; ok && group == api.ServerGroupDBServers {
-					newPlan = api.Plan{api.NewAction(api.ActionTypeMarkToRemoveMember, group, m.ID, "Replace flag present")}
-					continue
+				if _, ok := pod.Annotations[deployment.ArangoDeploymentPodReplaceAnnotation]; ok && (group == api.ServerGroupDBServers || group == api.ServerGroupAgents) {
+					if !m.Conditions.IsTrue(api.ConditionTypeMarkedToRemove) {
+						newPlan = api.Plan{api.NewAction(api.ActionTypeMarkToRemoveMember, group, m.ID, "Replace flag present")}
+						continue
+					}
 				}
 			}
 		}

pkg/deployment/reconcile/plan_builder_scale.go

@@ -111,18 +111,30 @@ func createReplaceMemberPlan(ctx context.Context,
 					return nil
 				}
 				if member.Conditions.IsTrue(api.ConditionTypeMarkedToRemove) {
-					plan = append(plan, api.NewAction(api.ActionTypeAddMember, group, "").
-						AddParam(api.ActionTypeWaitForMemberInSync.String(), "").
-						AddParam(api.ActionTypeWaitForMemberUp.String(), ""))
-					log.Debug().
-						Str("role", group.AsRole()).
-						Msg("Creating replacement plan")
-					return nil
+					switch group {
+					case api.ServerGroupDBServers:
+						plan = append(plan, api.NewAction(api.ActionTypeAddMember, group, "").
+							AddParam(api.ActionTypeWaitForMemberInSync.String(), "").
+							AddParam(api.ActionTypeWaitForMemberUp.String(), ""))
+						log.Debug().
+							Str("role", group.AsRole()).
+							Msg("Creating replacement plan")
+						return nil
+					case api.ServerGroupAgents:
+						plan = append(plan, api.NewAction(api.ActionTypeRemoveMember, group, member.ID),
+							api.NewAction(api.ActionTypeAddMember, group, "").
+								AddParam(api.ActionTypeWaitForMemberInSync.String(), "").
+								AddParam(api.ActionTypeWaitForMemberUp.String(), ""))
+						log.Debug().
+							Str("role", group.AsRole()).
+							Msg("Creating replacement plan")
+						return nil
+					}
 				}
 			}
 
 			return nil
-		}, api.ServerGroupDBServers)
+		}, api.ServerGroupAgents, api.ServerGroupDBServers)
 	}
 
 	return plan

pkg/util/k8sutil/names/id.go

@@ -0,0 +1,52 @@
+//
+// DISCLAIMER
+//
+// Copyright 2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package names
+
+import (
+	"fmt"
+	"strings"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/dchest/uniuri"
+)
+
+func GetArangodID(group api.ServerGroup) string {
+	return fmt.Sprintf("%s%s", GetArangodIDPrefix(group), strings.ToLower(uniuri.NewLen(8)))
+}
+
+// GetArangodIDPrefix returns the prefix required ID's of arangod servers
+// in the given group.
+func GetArangodIDPrefix(group api.ServerGroup) string {
+	switch group {
+	case api.ServerGroupSingle:
+		return "SNGL-"
+	case api.ServerGroupCoordinators:
+		return "CRDN-"
+	case api.ServerGroupDBServers:
+		return "PRMR-"
+	case api.ServerGroupAgents:
+		return "AGNT-"
+	default:
+		return ""
+	}
+}