GT-143 ArangoBackup: create retries and MaxIterations limit (#1269)

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Change Log
 
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
+- (Feature) ArangoBackup create retries and MaxIterations limit
 
 ## [1.2.27](https://github.com/arangodb/kube-arangodb/tree/1.2.27) (2023-04-27)
 - (Feature) Add InSync Cache

pkg/apis/backup/v1/backup_spec_backoff.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -29,6 +29,8 @@ type ArangoBackupSpecBackOff struct {
 	MaxDelay *int `json:"max_delay,omitempty"`
 	// Iterations defines number of iterations before reaching MaxDelay. Default to 5
 	Iterations *int `json:"iterations,omitempty"`
+	// MaxIterations defines maximum number of iterations after backoff will be disabled. Default to nil (no limit)
+	MaxIterations *int `json:"max_iterations,omitempty"`
 }
 
 func (a *ArangoBackupSpecBackOff) GetMaxDelay() int {

pkg/apis/backup/v1/backup_state.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -34,6 +34,7 @@ const (
 	ArangoBackupStateDownloadError state.State = "DownloadError"
 	ArangoBackupStateDownloading   state.State = "Downloading"
 	ArangoBackupStateCreate        state.State = "Create"
+	ArangoBackupStateCreateError   state.State = "CreateError"
 	ArangoBackupStateUpload        state.State = "Upload"
 	ArangoBackupStateUploading     state.State = "Uploading"
 	ArangoBackupStateUploadError   state.State = "UploadError"
@@ -50,7 +51,8 @@ var ArangoBackupStateMap = state.Map{
 	ArangoBackupStateDownload:      {ArangoBackupStateDownloading, ArangoBackupStateFailed, ArangoBackupStateDownloadError},
 	ArangoBackupStateDownloading:   {ArangoBackupStateReady, ArangoBackupStateFailed, ArangoBackupStateDownloadError},
 	ArangoBackupStateDownloadError: {ArangoBackupStatePending, ArangoBackupStateFailed},
-	ArangoBackupStateCreate:        {ArangoBackupStateReady, ArangoBackupStateFailed},
+	ArangoBackupStateCreate:        {ArangoBackupStateReady, ArangoBackupStateFailed, ArangoBackupStateCreateError},
+	ArangoBackupStateCreateError:   {ArangoBackupStateFailed, ArangoBackupStateCreate},
 	ArangoBackupStateUpload:        {ArangoBackupStateUploading, ArangoBackupStateFailed, ArangoBackupStateDeleted, ArangoBackupStateUploadError},
 	ArangoBackupStateUploading:     {ArangoBackupStateReady, ArangoBackupStateFailed, ArangoBackupStateUploadError},
 	ArangoBackupStateUploadError:   {ArangoBackupStateFailed, ArangoBackupStateReady},

pkg/apis/backup/v1/backup_status_backoff.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -51,7 +51,19 @@ func (a *ArangoBackupStatusBackOff) GetNext() meta.Time {
 	return a.Next
 }
 
+func (a *ArangoBackupStatusBackOff) ShouldBackoff(spec *ArangoBackupSpecBackOff) bool {
+	return spec == nil || spec.MaxIterations == nil || a.GetIterations() < *spec.MaxIterations
+}
+
 func (a *ArangoBackupStatusBackOff) Backoff(spec *ArangoBackupSpecBackOff) *ArangoBackupStatusBackOff {
+	if !a.ShouldBackoff(spec) {
+		// Do not backoff anymore
+		return &ArangoBackupStatusBackOff{
+			Iterations: a.GetIterations(),
+			Next:       a.GetNext(),
+		}
+	}
+
 	return &ArangoBackupStatusBackOff{
 		Iterations: a.GetIterations() + 1,
 		Next:       meta.Time{Time: time.Now().Add(spec.Backoff(a.GetIterations()))},

pkg/apis/backup/v1/backup_status_backoff_test.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -25,6 +25,8 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/arangodb/kube-arangodb/pkg/util"
 )
 
 func TestArangoBackupStatusBackOff_Backoff(t *testing.T) {
@@ -37,4 +39,21 @@ func TestArangoBackupStatusBackOff_Backoff(t *testing.T) {
 		require.Equal(t, 1, n.GetIterations())
 		require.True(t, n.GetNext().After(time.Now().Add(time.Duration(9.9*float64(time.Second)))))
 	})
+
+	t.Run("Test MaxIterations", func(t *testing.T) {
+		var spec = &ArangoBackupSpecBackOff{
+			Iterations:    util.NewInt(2),
+			MaxIterations: util.NewInt(3),
+		}
+		var status *ArangoBackupStatusBackOff
+
+		n := status.Backoff(spec)
+		require.Equal(t, 1, n.GetIterations())
+		require.True(t, n.ShouldBackoff(spec))
+
+		n.Iterations = 3
+		n = n.Backoff(spec)
+		require.Equal(t, 3, n.GetIterations())
+		require.False(t, n.ShouldBackoff(spec))
+	})
 }

pkg/apis/backup/v1/zz_generated.deepcopy.go

@@ -310,6 +310,11 @@ func (in *ArangoBackupSpecBackOff) DeepCopyInto(out *ArangoBackupSpecBackOff) {
 		*out = new(int)
 		**out = **in
 	}
+	if in.MaxIterations != nil {
+		in, out := &in.MaxIterations, &out.MaxIterations
+		*out = new(int)
+		**out = **in
+	}
 	return
 }
 

pkg/handlers/backup/state.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -33,6 +33,7 @@ var (
 		backupApi.ArangoBackupStatePending:       statePendingHandler,
 		backupApi.ArangoBackupStateScheduled:     stateScheduledHandler,
 		backupApi.ArangoBackupStateCreate:        stateCreateHandler,
+		backupApi.ArangoBackupStateCreateError:   stateCreateErrorHandler,
 		backupApi.ArangoBackupStateUpload:        stateUploadHandler,
 		backupApi.ArangoBackupStateUploading:     stateUploadingHandler,
 		backupApi.ArangoBackupStateUploadError:   stateUploadErrorHandler,

pkg/handlers/backup/state_create.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -21,6 +21,8 @@
 package backup
 
 import (
+	"time"
+
 	"github.com/arangodb/go-driver"
 
 	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
@@ -39,7 +41,12 @@ func stateCreateHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.
 
 	response, err := client.Create()
 	if err != nil {
-		return nil, err
+		return wrapUpdateStatus(backup,
+			updateStatusState(backupApi.ArangoBackupStateCreateError, "Create failed with error: %s", err.Error()),
+			cleanStatusJob(),
+			updateStatusAvailable(false),
+			addBackOff(backup.Spec),
+		)
 	}
 
 	backupMeta, err := client.Get(response.ID)
@@ -59,5 +66,25 @@ func stateCreateHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.
 		updateStatusState(backupApi.ArangoBackupStateReady, ""),
 		updateStatusAvailable(true),
 		updateStatusBackup(backupMeta),
+		cleanBackOff(),
 	)
 }
+
+func stateCreateErrorHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) {
+	// no more retries - move to failed state
+	if !backup.Status.Backoff.ShouldBackoff(backup.Spec.Backoff) {
+		return wrapUpdateStatus(backup,
+			updateStatusState(backupApi.ArangoBackupStateFailed, "out of Create retries"),
+			cleanStatusJob())
+	}
+
+	// if we should retry - move to create state
+	if backup.Status.Backoff.ShouldBackoff(backup.Spec.Backoff) && !backup.Status.Backoff.GetNext().After(time.Now()) {
+		return wrapUpdateStatus(backup,
+			updateStatusState(backupApi.ArangoBackupStateCreate, ""),
+			cleanStatusJob())
+	}
+
+	// no ready to retry - wait (do not change state)
+	return wrapUpdateStatus(backup)
+}

pkg/handlers/backup/state_create_test.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -22,8 +22,10 @@ package backup
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/arangodb/go-driver"
 
@@ -120,7 +122,7 @@ func Test_State_Create_Upload(t *testing.T) {
 	compareBackupMeta(t, backupMeta, newObj)
 }
 
-func Test_State_Create_CreateFailed(t *testing.T) {
+func Test_State_Create_CreateError(t *testing.T) {
 	// Arrange
 	error := newFatalErrorf("error")
 	handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{
@@ -137,32 +139,75 @@ func Test_State_Create_CreateFailed(t *testing.T) {
 
 	// Assert
 	newObj := refreshArangoBackup(t, handler, obj)
-	require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateFailed)
-	require.Equal(t, newObj.Status.Message, createStateMessage(backupApi.ArangoBackupStateCreate, backupApi.ArangoBackupStateFailed, error.Error()))
-
+	require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateCreateError)
 	require.Nil(t, newObj.Status.Backup)
-
 	require.False(t, newObj.Status.Available)
 }
 
-func Test_State_Create_TemporaryCreateFailed(t *testing.T) {
+func Test_State_CreateError_Retry(t *testing.T) {
 	// Arrange
-	error := newTemporaryErrorf("error")
-	handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{
-		createError: error,
-	})
+	handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{})
 
-	obj, deployment := newObjectSet(backupApi.ArangoBackupStateCreate)
+	obj, deployment := newObjectSet(backupApi.ArangoBackupStateCreateError)
+
+	backupMeta, err := mock.Create()
+	require.NoError(t, err)
+
+	obj.Status.Backup = &backupApi.ArangoBackupDetails{
+		ID:                string(backupMeta.ID),
+		Version:           backupMeta.Version,
+		CreationTimestamp: meta.Now(),
+	}
+
+	obj.Status.Time.Time = time.Now().Add(-2 * downloadDelay)
 
 	// Act
 	createArangoDeployment(t, handler, deployment)
 	createArangoBackup(t, handler, obj)
 
-	err := handler.Handle(newItemFromBackup(operation.Update, obj))
-	require.EqualError(t, err, error.Error())
+	require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj)))
 
 	// Assert
 	newObj := refreshArangoBackup(t, handler, obj)
-
-	require.Equal(t, obj.Status, newObj.Status)
+	require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateCreate)
+	require.False(t, newObj.Status.Available)
+	require.NotNil(t, newObj.Status.Backup)
+	require.Equal(t, obj.Status.Backup, newObj.Status.Backup)
+}
+
+func Test_State_CreateError_Transfer_To_Failed(t *testing.T) {
+	// Arrange
+	handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{})
+
+	obj, deployment := newObjectSet(backupApi.ArangoBackupStateCreateError)
+
+	backupMeta, err := mock.Create()
+	require.NoError(t, err)
+
+	obj.Status.Backup = &backupApi.ArangoBackupDetails{
+		ID:                string(backupMeta.ID),
+		Version:           backupMeta.Version,
+		CreationTimestamp: meta.Now(),
+	}
+	obj.Status.Backoff = &backupApi.ArangoBackupStatusBackOff{
+		Iterations: 2,
+	}
+
+	obj.Spec.Backoff = &backupApi.ArangoBackupSpecBackOff{
+		Iterations:    util.NewInt(1),
+		MaxIterations: util.NewInt(2),
+	}
+
+	obj.Status.Time.Time = time.Now().Add(-2 * downloadDelay)
+
+	// Act
+	createArangoDeployment(t, handler, deployment)
+	createArangoBackup(t, handler, obj)
+
+	require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj)))
+
+	// Assert
+	newObj := refreshArangoBackup(t, handler, obj)
+	require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateFailed)
+	require.Equal(t, newObj.Status.Message, "out of Create retries")
 }

pkg/handlers/backup/state_uploaderror.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,13 +27,23 @@ import (
 )
 
 func stateUploadErrorHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) {
-	if backup.Spec.Upload == nil || !backup.Status.Backoff.GetNext().After(time.Now()) {
+	// no more retries - move to failed state
+	if !backup.Status.Backoff.ShouldBackoff(backup.Spec.Backoff) {
+		return wrapUpdateStatus(backup,
+			updateStatusState(backupApi.ArangoBackupStateFailed, "out of Upload retries"),
+			cleanStatusJob())
+	}
+
+	// if we should retry - move to ready state
+	if backup.Spec.Upload == nil ||
+		(backup.Status.Backoff.ShouldBackoff(backup.Spec.Backoff) && !backup.Status.Backoff.GetNext().After(time.Now())) {
 		return wrapUpdateStatus(backup,
 			updateStatusState(backupApi.ArangoBackupStateReady, ""),
 			cleanStatusJob(),
 			updateStatusAvailable(true))
 	}
 
+	// no ready to retry - wait (do not change state)
 	return wrapUpdateStatus(backup,
 		updateStatusAvailable(true))
 }