[Feature] Add upgrade steps procedure (#933)

CHANGELOG.md

@@ -13,6 +13,7 @@
 - (Bugfix) Fix NPE in State fetcher
 - (Refactor) Configurable throttle inspector
 - (Bugfix) Skip Replace operation on DBServer if they need to be scaled down
+- (Feature) Upgrade procedure steps
 
 ## [1.2.8](https://github.com/arangodb/kube-arangodb/tree/1.2.8) (2022-02-24)
 - Do not check License V2 on Community images

Makefile

@@ -477,6 +477,7 @@ set-api-version/%:
 	      "$(ROOT)/pkg/util/" \
 	      "$(ROOT)/pkg/handlers/" \
 	      "$(ROOT)/pkg/apis/backup/" \
+	      "$(ROOT)/pkg/upgrade/" \
 	  | cut -d ':' -f 1 | sort | uniq \
 	  | xargs -n 1 sed -i "s#github.com/arangodb/kube-arangodb/pkg/apis/$*/v[A-Za-z0-9]\+#github.com/arangodb/kube-arangodb/pkg/apis/$*/v$(API_VERSION)#g"
 	@grep -rHn "DatabaseV[A-Za-z0-9]\+()" \
@@ -487,6 +488,7 @@ set-api-version/%:
 		  "$(ROOT)/pkg/util/" \
 	      "$(ROOT)/pkg/handlers/" \
 	      "$(ROOT)/pkg/apis/backup/" \
+	      "$(ROOT)/pkg/upgrade/" \
 	  | cut -d ':' -f 1 | sort | uniq \
 	  | xargs -n 1 sed -i "s#DatabaseV[A-Za-z0-9]\+()\.#DatabaseV$(API_VERSION)().#g"
 	@grep -rHn "ReplicationV[A-Za-z0-9]\+()" \
@@ -497,6 +499,7 @@ set-api-version/%:
 		  "$(ROOT)/pkg/util/" \
 		  "$(ROOT)/pkg/handlers" \
 		  "$(ROOT)/pkg/apis/backup/" \
+	      "$(ROOT)/pkg/upgrade/" \
 	  | cut -d ':' -f 1 | sort | uniq \
 	  | xargs -n 1 sed -i "s#ReplicationV[A-Za-z0-9]\+()\.#ReplicationV$(API_VERSION)().#g"
 

pkg/apis/deployment/v1/deployment_status.go

@@ -85,6 +85,8 @@ type DeploymentStatus struct {
 	Rebalancer *ArangoDeploymentRebalancerStatus `json:"rebalancer,omitempty"`
 
 	BackOff BackOff `json:"backoff,omitempty"`
+
+	Version *Version `json:"version,omitempty"`
 }
 
 // Equal checks for equality
@@ -105,7 +107,8 @@ func (ds *DeploymentStatus) Equal(other DeploymentStatus) bool {
 		ds.SecretHashes.Equal(other.SecretHashes) &&
 		ds.Agency.Equal(other.Agency) &&
 		ds.Topology.Equal(other.Topology) &&
-		ds.BackOff.Equal(other.BackOff)
+		ds.BackOff.Equal(other.BackOff) &&
+		ds.Version.Equal(other.Version)
 }
 
 // IsForceReload returns true if ForceStatusReload is set to true

pkg/apis/deployment/v1/version.go

@@ -0,0 +1,161 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+var _ json.Marshaler = Version{}
+var _ json.Unmarshaler = &Version{}
+
+type Version struct {
+	Major int `json:"major"`
+	Minor int `json:"minor"`
+	Patch int `json:"patch"`
+	ID    int `json:"ID,omitempty"`
+}
+
+func (v Version) Compare(b Version) int {
+	if v.Major > b.Major {
+		return 1
+	} else if v.Major < b.Major {
+		return -1
+	}
+
+	if v.Minor > b.Minor {
+		return 1
+	} else if v.Minor < b.Minor {
+		return -1
+	}
+
+	if v.Patch > b.Patch {
+		return 1
+	} else if v.Patch < b.Patch {
+		return -1
+	}
+
+	if v.ID < b.ID {
+		return 1
+	} else if b.ID < v.ID {
+		return -1
+	}
+
+	return 0
+}
+
+func (v *Version) Equal(b *Version) bool {
+	if v == nil && b == nil {
+		return true
+	}
+	if v == nil || b == nil {
+		return true
+	}
+	return v.Major == b.Major && v.Minor == b.Minor && v.Patch == b.Patch && v.ID == b.ID
+}
+
+func (v *Version) UnmarshalJSON(bytes []byte) error {
+	if v == nil {
+		return errors.Errorf("Nil version provided")
+	}
+
+	var s string
+
+	if err := json.Unmarshal(bytes, &s); err != nil {
+		*v = Version{
+			Major: 0,
+			Minor: 0,
+			Patch: 0,
+		}
+		return nil
+	}
+
+	z := strings.Split(s, ".")
+
+	i := make([]int, len(z))
+
+	for id, z := range z {
+		if q, err := strconv.Atoi(z); err != nil {
+			*v = Version{
+				Major: 0,
+				Minor: 0,
+				Patch: 0,
+			}
+			return nil
+		} else {
+			i[id] = q
+		}
+	}
+	switch l := len(i); l {
+	case 1:
+		var n Version
+
+		n.Major = i[0]
+
+		*v = n
+	case 2:
+		var n Version
+
+		n.Major = i[0]
+		n.Minor = i[1]
+
+		*v = n
+	case 3:
+		var n Version
+
+		n.Major = i[0]
+		n.Minor = i[1]
+		n.Patch = i[2]
+
+		*v = n
+	case 4:
+		var n Version
+
+		n.Major = i[0]
+		n.Minor = i[1]
+		n.Patch = i[2]
+		n.ID = i[3]
+
+		*v = n
+	default:
+		*v = Version{
+			Major: 0,
+			Minor: 0,
+			Patch: 0,
+		}
+		return nil
+	}
+
+	return nil
+}
+
+func (v Version) MarshalJSON() ([]byte, error) {
+	if v.ID == 0 {
+		return json.Marshal(fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch))
+	}
+
+	return json.Marshal(fmt.Sprintf("%d.%d.%d.%d", v.Major, v.Minor, v.Patch, v.ID))
+}

pkg/apis/deployment/v1/version_test.go

@@ -0,0 +1,57 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func remarshalVersionWithExpected(t *testing.T, version, expected string) {
+	t.Run(version, func(t *testing.T) {
+		var v Version
+
+		data, err := json.Marshal(version)
+		require.NoError(t, err)
+
+		require.NoError(t, json.Unmarshal(data, &v))
+
+		data, err = json.Marshal(v)
+		require.NoError(t, err)
+
+		var newV string
+
+		require.NoError(t, json.Unmarshal(data, &newV))
+
+		require.Equal(t, expected, newV)
+	})
+}
+
+func Test_Version(t *testing.T) {
+	remarshalVersionWithExpected(t, "1", "1.0.0")
+	remarshalVersionWithExpected(t, "1.0", "1.0.0")
+	remarshalVersionWithExpected(t, "1.0.0", "1.0.0")
+	remarshalVersionWithExpected(t, "1.0.0.0", "1.0.0")
+	remarshalVersionWithExpected(t, "1.0.0.1", "1.0.0.1")
+	remarshalVersionWithExpected(t, "Invalid", "0.0.0")
+}

pkg/apis/deployment/v1/zz_generated.deepcopy.go

@@ -1159,6 +1159,11 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) {
 			(*out)[key] = *val.DeepCopy()
 		}
 	}
+	if in.Version != nil {
+		in, out := &in.Version, &out.Version
+		*out = new(Version)
+		**out = **in
+	}
 	return
 }
 
@@ -2986,3 +2991,19 @@ func (in TopologyStatusZones) DeepCopy() TopologyStatusZones {
 	in.DeepCopyInto(out)
 	return *out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Version) DeepCopyInto(out *Version) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Version.
+func (in *Version) DeepCopy() *Version {
+	if in == nil {
+		return nil
+	}
+	out := new(Version)
+	in.DeepCopyInto(out)
+	return out
+}

pkg/apis/deployment/v2alpha1/deployment_status.go

@@ -85,6 +85,8 @@ type DeploymentStatus struct {
 	Rebalancer *ArangoDeploymentRebalancerStatus `json:"rebalancer,omitempty"`
 
 	BackOff BackOff `json:"backoff,omitempty"`
+
+	Version *Version `json:"version,omitempty"`
 }
 
 // Equal checks for equality
@@ -105,7 +107,8 @@ func (ds *DeploymentStatus) Equal(other DeploymentStatus) bool {
 		ds.SecretHashes.Equal(other.SecretHashes) &&
 		ds.Agency.Equal(other.Agency) &&
 		ds.Topology.Equal(other.Topology) &&
-		ds.BackOff.Equal(other.BackOff)
+		ds.BackOff.Equal(other.BackOff) &&
+		ds.Version.Equal(other.Version)
 }
 
 // IsForceReload returns true if ForceStatusReload is set to true

pkg/apis/deployment/v2alpha1/version.go

@@ -0,0 +1,161 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v2alpha1
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+var _ json.Marshaler = Version{}
+var _ json.Unmarshaler = &Version{}
+
+type Version struct {
+	Major int `json:"major"`
+	Minor int `json:"minor"`
+	Patch int `json:"patch"`
+	ID    int `json:"ID,omitempty"`
+}
+
+func (v Version) Compare(b Version) int {
+	if v.Major > b.Major {
+		return 1
+	} else if v.Major < b.Major {
+		return -1
+	}
+
+	if v.Minor > b.Minor {
+		return 1
+	} else if v.Minor < b.Minor {
+		return -1
+	}
+
+	if v.Patch > b.Patch {
+		return 1
+	} else if v.Patch < b.Patch {
+		return -1
+	}
+
+	if v.ID < b.ID {
+		return 1
+	} else if b.ID < v.ID {
+		return -1
+	}
+
+	return 0
+}
+
+func (v *Version) Equal(b *Version) bool {
+	if v == nil && b == nil {
+		return true
+	}
+	if v == nil || b == nil {
+		return true
+	}
+	return v.Major == b.Major && v.Minor == b.Minor && v.Patch == b.Patch && v.ID == b.ID
+}
+
+func (v *Version) UnmarshalJSON(bytes []byte) error {
+	if v == nil {
+		return errors.Errorf("Nil version provided")
+	}
+
+	var s string
+
+	if err := json.Unmarshal(bytes, &s); err != nil {
+		*v = Version{
+			Major: 0,
+			Minor: 0,
+			Patch: 0,
+		}
+		return nil
+	}
+
+	z := strings.Split(s, ".")
+
+	i := make([]int, len(z))
+
+	for id, z := range z {
+		if q, err := strconv.Atoi(z); err != nil {
+			*v = Version{
+				Major: 0,
+				Minor: 0,
+				Patch: 0,
+			}
+			return nil
+		} else {
+			i[id] = q
+		}
+	}
+	switch l := len(i); l {
+	case 1:
+		var n Version
+
+		n.Major = i[0]
+
+		*v = n
+	case 2:
+		var n Version
+
+		n.Major = i[0]
+		n.Minor = i[1]
+
+		*v = n
+	case 3:
+		var n Version
+
+		n.Major = i[0]
+		n.Minor = i[1]
+		n.Patch = i[2]
+
+		*v = n
+	case 4:
+		var n Version
+
+		n.Major = i[0]
+		n.Minor = i[1]
+		n.Patch = i[2]
+		n.ID = i[3]
+
+		*v = n
+	default:
+		*v = Version{
+			Major: 0,
+			Minor: 0,
+			Patch: 0,
+		}
+		return nil
+	}
+
+	return nil
+}
+
+func (v Version) MarshalJSON() ([]byte, error) {
+	if v.ID == 0 {
+		return json.Marshal(fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch))
+	}
+
+	return json.Marshal(fmt.Sprintf("%d.%d.%d.%d", v.Major, v.Minor, v.Patch, v.ID))
+}

pkg/apis/deployment/v2alpha1/version_test.go

@@ -0,0 +1,57 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v2alpha1
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func remarshalVersionWithExpected(t *testing.T, version, expected string) {
+	t.Run(version, func(t *testing.T) {
+		var v Version
+
+		data, err := json.Marshal(version)
+		require.NoError(t, err)
+
+		require.NoError(t, json.Unmarshal(data, &v))
+
+		data, err = json.Marshal(v)
+		require.NoError(t, err)
+
+		var newV string
+
+		require.NoError(t, json.Unmarshal(data, &newV))
+
+		require.Equal(t, expected, newV)
+	})
+}
+
+func Test_Version(t *testing.T) {
+	remarshalVersionWithExpected(t, "1", "1.0.0")
+	remarshalVersionWithExpected(t, "1.0", "1.0.0")
+	remarshalVersionWithExpected(t, "1.0.0", "1.0.0")
+	remarshalVersionWithExpected(t, "1.0.0.0", "1.0.0")
+	remarshalVersionWithExpected(t, "1.0.0.1", "1.0.0.1")
+	remarshalVersionWithExpected(t, "Invalid", "0.0.0")
+}

pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go

@@ -1159,6 +1159,11 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) {
 			(*out)[key] = *val.DeepCopy()
 		}
 	}
+	if in.Version != nil {
+		in, out := &in.Version, &out.Version
+		*out = new(Version)
+		**out = **in
+	}
 	return
 }
 
@@ -2986,3 +2991,19 @@ func (in TopologyStatusZones) DeepCopy() TopologyStatusZones {
 	in.DeepCopyInto(out)
 	return *out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Version) DeepCopyInto(out *Version) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Version.
+func (in *Version) DeepCopy() *Version {
+	if in == nil {
+		return nil
+	}
+	out := new(Version)
+	in.DeepCopyInto(out)
+	return out
+}

pkg/deployment/deployment_inspector.go

@@ -41,6 +41,7 @@ import (
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/acs"
 	"github.com/arangodb/kube-arangodb/pkg/metrics"
+	"github.com/arangodb/kube-arangodb/pkg/upgrade"
 	"github.com/arangodb/kube-arangodb/pkg/util"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -116,6 +117,18 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval
 
 		d.GetMembersState().RefreshState(ctxReconciliation, updated.Status.Members.AsList())
 		d.GetMembersState().Log(d.deps.Log)
+		if err := d.WithStatusUpdateErr(ctxReconciliation, func(s *api.DeploymentStatus) (bool, error) {
+			if changed, err := upgrade.RunUpgrade(*updated, s, d.GetCachedStatus()); err != nil {
+				return false, err
+			} else {
+				return changed, nil
+			}
+		}); err != nil {
+			d.CreateEvent(k8sutil.NewErrorEvent("Upgrade failed", err, d.apiObject))
+			nextInterval = minInspectionInterval
+			d.recentInspectionErrors++
+			return nextInterval.ReduceTo(maxInspectionInterval)
+		}
 
 		inspectNextInterval, err := d.inspectDeploymentWithError(ctxReconciliation, nextInterval)
 		if err != nil {

pkg/upgrade/member_cid_append.go

@@ -0,0 +1,50 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package upgrade
+
+import (
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/interfaces"
+)
+
+func init() {
+	registerUpgrade(memberCIDAppend())
+}
+
+func memberCIDAppend() Upgrade {
+	return newUpgrade(api.Version{
+		Major: 1,
+		Minor: 2,
+		Patch: 8,
+		ID:    1,
+	}, func(obj api.ArangoDeployment, status *api.DeploymentStatus, _ interfaces.Inspector) error {
+		for _, i := range status.Members.AsList() {
+			if i.Member.ClusterID == "" {
+				i.Member.ClusterID = obj.GetUID()
+				if err := status.Members.Update(i.Member, i.Group); err != nil {
+					return err
+				}
+			}
+		}
+
+		return nil
+	})
+}

pkg/upgrade/member_cid_append_test.go

@@ -0,0 +1,49 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package upgrade
+
+import (
+	"testing"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/util/uuid"
+)
+
+func testMemberCIDAppendPrepare(t *testing.T, obj *api.ArangoDeployment) {
+	t.Run("Member CID Append", func(t *testing.T) {
+		obj.UID = uuid.NewUUID()
+
+		obj.Status.Members.Agents = append(obj.Status.Members.Agents, api.MemberStatus{
+			ID:        "CIDAppend",
+			ClusterID: "",
+		})
+	})
+}
+
+func testMemberCIDAppendCheck(t *testing.T, obj api.ArangoDeployment) {
+	t.Run("Member CID Append", func(t *testing.T) {
+		m, g, ok := obj.Status.Members.ElementByID("CIDAppend")
+		require.True(t, ok)
+		require.Equal(t, api.ServerGroupAgents, g)
+		require.Equal(t, obj.GetUID(), m.ClusterID)
+	})
+}

pkg/upgrade/upgrade.go

@@ -0,0 +1,149 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package upgrade
+
+import (
+	"sort"
+	"sync"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/interfaces"
+	"github.com/pkg/errors"
+)
+
+var (
+	upgradeLock sync.Mutex
+	upgrades    Upgrades
+)
+
+func registerUpgrade(u Upgrade) {
+	upgradeLock.Lock()
+	defer upgradeLock.Unlock()
+
+	upgrades = append(upgrades, u)
+}
+
+func RunUpgrade(obj api.ArangoDeployment, status *api.DeploymentStatus, cache interfaces.Inspector) (bool, error) {
+	upgradeLock.Lock()
+	defer upgradeLock.Unlock()
+
+	if changed, err := upgrades.Execute(obj, status, cache); err != nil {
+		return false, err
+	} else if changed {
+		v := upgrades[len(upgrades)-1].Version()
+		status.Version = &v
+		return true, nil
+	} else {
+		return false, nil
+	}
+}
+
+type Upgrades []Upgrade
+
+func (u Upgrades) Execute(obj api.ArangoDeployment, status *api.DeploymentStatus, cache interfaces.Inspector) (bool, error) {
+	z := u.Sort()
+
+	if err := z.Verify(); err != nil {
+		return false, err
+	}
+
+	var v api.Version
+	if status != nil && status.Version != nil {
+		v = *status.Version
+	}
+
+	var changed bool
+
+	for _, up := range z {
+		if up.Version().Compare(v) < 0 {
+			continue
+		}
+		changed = true
+		if err := up.ArangoDeployment(obj, status, cache); err != nil {
+			return false, err
+		}
+	}
+
+	return changed, nil
+}
+
+func (u Upgrades) Verify() error {
+	v := map[int]map[int]map[int][]int{}
+
+	for _, z := range u {
+		ver := z.Version()
+
+		l1 := v[ver.Major]
+		if l1 == nil {
+			l1 = map[int]map[int][]int{}
+		}
+
+		l2 := l1[ver.Minor]
+		if l2 == nil {
+			l2 = map[int][]int{}
+		}
+
+		l3 := l2[ver.Patch]
+
+		l3 = append(l3, ver.ID)
+
+		l2[ver.Patch] = l3
+
+		l1[ver.Minor] = l2
+
+		v[ver.Major] = l1
+	}
+
+	for major, majorV := range v {
+		for minor, minorV := range majorV {
+			for patch, patchV := range minorV {
+				for id := range patchV {
+					if id+1 != patchV[id] {
+						return errors.Errorf("Invalid version in %d.%d.%d - got %d, expected %d", major, minor, patch, patchV[id], id+1)
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (u Upgrades) Copy() Upgrades {
+	c := make(Upgrades, len(u))
+
+	copy(c, u)
+
+	return c
+}
+
+func (u Upgrades) Sort() Upgrades {
+	sort.Slice(u, func(i, j int) bool {
+		return u[i].Version().Compare(u[j].Version()) < 0
+	})
+	return u
+}
+
+type Upgrade interface {
+	Version() api.Version
+
+	ArangoDeployment(obj api.ArangoDeployment, status *api.DeploymentStatus, cache interfaces.Inspector) error
+}

pkg/upgrade/upgrade_struct.go

@@ -0,0 +1,51 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package upgrade
+
+import (
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/interfaces"
+)
+
+func newUpgrade(version api.Version, u func(obj api.ArangoDeployment, status *api.DeploymentStatus, cache interfaces.Inspector) error) Upgrade {
+	return &upgrade{
+		version: version,
+		upgrade: u,
+	}
+}
+
+type upgrade struct {
+	version api.Version
+
+	upgrade func(obj api.ArangoDeployment, status *api.DeploymentStatus, cache interfaces.Inspector) error
+}
+
+func (u upgrade) Version() api.Version {
+	return u.version
+}
+
+func (u upgrade) ArangoDeployment(obj api.ArangoDeployment, status *api.DeploymentStatus, cache interfaces.Inspector) error {
+	if q := u.upgrade; q != nil {
+		return q(obj, status, cache)
+	}
+
+	return nil
+}

pkg/upgrade/upgrade_test.go

@@ -0,0 +1,227 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package upgrade
+
+import (
+	"testing"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/interfaces"
+	"github.com/arangodb/kube-arangodb/pkg/util/kclient"
+	"github.com/arangodb/kube-arangodb/pkg/util/tests"
+	"github.com/stretchr/testify/require"
+)
+
+func genNewVersionAppender(v *[]api.Version, version api.Version) Upgrade {
+	return newUpgrade(version, func(obj api.ArangoDeployment, status *api.DeploymentStatus, cache interfaces.Inspector) error {
+		*v = append(*v, version)
+		return nil
+	})
+}
+
+func Test_Verify(t *testing.T) {
+	require.NoError(t, upgrades.Verify())
+}
+
+func Test_Verify_WrongOrder(t *testing.T) {
+	t.Run("Invalid version - starts from 0", func(t *testing.T) {
+		var u Upgrades
+
+		var v []api.Version
+
+		u = append(u, genNewVersionAppender(&v, api.Version{
+			Major: 1,
+			Minor: 1,
+			Patch: 1,
+			ID:    0,
+		}))
+
+		_, err := u.Execute(api.ArangoDeployment{}, nil, nil)
+		require.EqualError(t, err, "Invalid version in 1.1.1 - got 0, expected 1")
+	})
+	t.Run("Invalid version - missing middle", func(t *testing.T) {
+		var u Upgrades
+
+		var v []api.Version
+
+		u = append(u,
+			genNewVersionAppender(&v, api.Version{
+				Major: 1,
+				Minor: 1,
+				Patch: 1,
+				ID:    1,
+			}),
+			genNewVersionAppender(&v, api.Version{
+				Major: 1,
+				Minor: 1,
+				Patch: 1,
+				ID:    3,
+			}),
+		)
+
+		_, err := u.Execute(api.ArangoDeployment{}, nil, nil)
+		require.EqualError(t, err, "Invalid version in 1.1.1 - got 3, expected 2")
+	})
+	t.Run("Valid multi version", func(t *testing.T) {
+		var u Upgrades
+
+		var v []api.Version
+
+		u = append(u,
+			genNewVersionAppender(&v, api.Version{
+				Major: 1,
+				Minor: 1,
+				Patch: 1,
+				ID:    1,
+			}),
+			genNewVersionAppender(&v, api.Version{
+				Major: 1,
+				Minor: 1,
+				Patch: 2,
+				ID:    1,
+			}),
+		)
+
+		_, err := u.Execute(api.ArangoDeployment{}, nil, nil)
+		require.NoError(t, err)
+		require.Len(t, v, 2)
+		require.Equal(t, api.Version{
+			Major: 1,
+			Minor: 1,
+			Patch: 1,
+			ID:    1,
+		}, v[0])
+		require.Equal(t, api.Version{
+			Major: 1,
+			Minor: 1,
+			Patch: 2,
+			ID:    1,
+		}, v[1])
+	})
+	t.Run("Valid multi version - rev order", func(t *testing.T) {
+		var u Upgrades
+
+		var v []api.Version
+
+		u = append(u,
+			genNewVersionAppender(&v, api.Version{
+				Major: 1,
+				Minor: 1,
+				Patch: 2,
+				ID:    1,
+			}),
+			genNewVersionAppender(&v, api.Version{
+				Major: 1,
+				Minor: 1,
+				Patch: 1,
+				ID:    1,
+			}),
+		)
+
+		_, err := u.Execute(api.ArangoDeployment{}, nil, nil)
+		require.NoError(t, err)
+		require.Len(t, v, 2)
+		require.Equal(t, api.Version{
+			Major: 1,
+			Minor: 1,
+			Patch: 1,
+			ID:    1,
+		}, v[0])
+		require.Equal(t, api.Version{
+			Major: 1,
+			Minor: 1,
+			Patch: 2,
+			ID:    1,
+		}, v[1])
+	})
+	t.Run("Valid multi version - only upgrade", func(t *testing.T) {
+		var u Upgrades
+
+		var v []api.Version
+
+		obj := api.ArangoDeployment{
+			Status: api.DeploymentStatus{
+				Version: &api.Version{
+					Major: 1,
+					Minor: 1,
+					Patch: 2,
+					ID:    1,
+				},
+			},
+		}
+
+		u = append(u,
+			genNewVersionAppender(&v, api.Version{
+				Major: 1,
+				Minor: 1,
+				Patch: 2,
+				ID:    1,
+			}),
+			genNewVersionAppender(&v, api.Version{
+				Major: 1,
+				Minor: 1,
+				Patch: 1,
+				ID:    1,
+			}),
+		)
+
+		status := obj.Status.DeepCopy()
+
+		_, err := u.Execute(obj, status, nil)
+		require.NoError(t, err)
+		require.Len(t, v, 1)
+		require.Equal(t, api.Version{
+			Major: 1,
+			Minor: 1,
+			Patch: 2,
+			ID:    1,
+		}, v[0])
+	})
+}
+
+func Test_RunUpgrade(t *testing.T) {
+	obj := api.ArangoDeployment{}
+
+	t.Run("Prepare", func(t *testing.T) {
+		testMemberCIDAppendPrepare(t, &obj)
+	})
+
+	status := obj.Status.DeepCopy()
+
+	c := kclient.NewFakeClient()
+
+	i := tests.NewInspector(t, c)
+
+	t.Run("Upgrade", func(t *testing.T) {
+		changed, err := RunUpgrade(obj, status, i)
+		require.NoError(t, err)
+		require.True(t, changed)
+	})
+
+	obj.Status = *status
+
+	t.Run("Check", func(t *testing.T) {
+		testMemberCIDAppendCheck(t, obj)
+	})
+
+	require.NotNil(t, obj.Status.Version)
+	require.Equal(t, upgrades[len(upgrades)-1].Version(), *obj.Status.Version)
+}

pkg/util/int.go

@@ -20,6 +20,22 @@
 
 package util
 
+func CompareInt(a, b int) bool {
+	return a == b
+}
+
+func CompareIntp(a, b *int) bool {
+	if a == nil && b == nil {
+		return true
+	}
+
+	if a == nil || b == nil {
+		return false
+	}
+
+	return CompareInt(*a, *b)
+}
+
 func CompareInt64(a, b int64) bool {
 	return a == b
 }