[Feature] [Backup] Backoff logic (#844)

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Change Log
 
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
+- Add ArangoBackup backoff functionality
 
 ## [1.2.5](https://github.com/arangodb/kube-arangodb/tree/1.2.5) (2021-10-25)
 - Split & Unify Lifecycle management functionality

pkg/apis/backup/v1/backup.go

@@ -17,8 +17,6 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
-// Author Adam Janikowski
-//
 
 package v1
 

pkg/apis/backup/v1/backup_policy.go

@@ -17,8 +17,6 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
-// Author Adam Janikowski
-//
 
 package v1
 

pkg/apis/backup/v1/backup_policy_spec.go

@@ -17,8 +17,6 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
-// Author Adam Janikowski
-//
 
 package v1
 

pkg/apis/backup/v1/backup_policy_status.go

@@ -17,8 +17,6 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
-// Author Adam Janikowski
-//
 
 package v1
 

pkg/apis/backup/v1/backup_policy_validate.go

@@ -17,8 +17,6 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
-// Author Adam Janikowski
-//
 
 package v1
 

pkg/apis/backup/v1/backup_spec.go

@@ -17,8 +17,6 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
-// Author Adam Janikowski
-//
 
 package v1
 
@@ -35,6 +33,8 @@ type ArangoBackupSpec struct {
 	Upload *ArangoBackupSpecOperation `json:"upload,omitempty"`
 
 	PolicyName *string `json:"policyName,omitempty"`
+
+	Backoff *ArangoBackupSpecBackOff `json:"backoff,omitempty"`
 }
 
 type ArangoBackupSpecDeployment struct {

pkg/apis/backup/v1/backup_spec_backoff.go

@@ -0,0 +1,92 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+import "time"
+
+type ArangoBackupSpecBackOff struct {
+	// MinDelay defines minimum delay in seconds. Default to 30
+	MinDelay *int `json:"min_delay,omitempty"`
+	// MaxDelay defines maximum delay in seconds. Default to 600
+	MaxDelay *int `json:"max_delay,omitempty"`
+	// Iterations defines number of iterations before reaching MaxDelay. Default to 5
+	Iterations *int `json:"iterations,omitempty"`
+}
+
+func (a *ArangoBackupSpecBackOff) GetMaxDelay() int {
+	if a == nil || a.MaxDelay == nil {
+		return 600
+	}
+
+	v := *a.MaxDelay
+
+	if v < 0 {
+		return 0
+	}
+
+	return v
+}
+
+func (a *ArangoBackupSpecBackOff) GetMinDelay() int {
+	if a == nil || a.MinDelay == nil {
+		return 30
+	}
+
+	v := *a.MinDelay
+
+	if v < 0 {
+		return 0
+	}
+
+	if m := a.GetMaxDelay(); m < v {
+		return m
+	}
+
+	return v
+}
+
+func (a *ArangoBackupSpecBackOff) GetIterations() int {
+	if a == nil || a.Iterations == nil {
+		return 5
+	}
+
+	v := *a.Iterations
+
+	if v < 1 {
+		return 1
+	}
+
+	return v
+}
+
+func (a *ArangoBackupSpecBackOff) Backoff(iteration int) time.Duration {
+	if maxIterations := a.GetIterations(); maxIterations <= iteration {
+		return time.Duration(a.GetMaxDelay()) * time.Second
+	} else {
+		min, max := a.GetMinDelay(), a.GetMaxDelay()
+
+		if min == max {
+			return time.Duration(min) * time.Second
+		}
+
+		return time.Duration(min+int(float64(iteration)/float64(maxIterations)*float64(max-min))) * time.Second
+	}
+}

pkg/apis/backup/v1/backup_spec_backoff_test.go

@@ -0,0 +1,89 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/arangodb/kube-arangodb/pkg/util"
+)
+
+func TestArangoBackupSpecBackOff_Backoff(t *testing.T) {
+	t.Run("Default", func(t *testing.T) {
+		var b *ArangoBackupSpecBackOff
+
+		assert.Equal(t, 30*time.Second, b.Backoff(0))
+		assert.Equal(t, 144*time.Second, b.Backoff(1))
+		assert.Equal(t, 258*time.Second, b.Backoff(2))
+		assert.Equal(t, 372*time.Second, b.Backoff(3))
+		assert.Equal(t, 486*time.Second, b.Backoff(4))
+		assert.Equal(t, 600*time.Second, b.Backoff(5))
+		assert.Equal(t, 600*time.Second, b.Backoff(6))
+	})
+	t.Run("Custom", func(t *testing.T) {
+		b := &ArangoBackupSpecBackOff{
+			MinDelay:   util.NewInt(20),
+			MaxDelay:   util.NewInt(120),
+			Iterations: util.NewInt(10),
+		}
+
+		assert.Equal(t, 20*time.Second, b.Backoff(0))
+		assert.Equal(t, 30*time.Second, b.Backoff(1))
+		assert.Equal(t, 40*time.Second, b.Backoff(2))
+		assert.Equal(t, 50*time.Second, b.Backoff(3))
+		assert.Equal(t, 60*time.Second, b.Backoff(4))
+		assert.Equal(t, 70*time.Second, b.Backoff(5))
+		assert.Equal(t, 80*time.Second, b.Backoff(6))
+		assert.Equal(t, 90*time.Second, b.Backoff(7))
+		assert.Equal(t, 100*time.Second, b.Backoff(8))
+		assert.Equal(t, 110*time.Second, b.Backoff(9))
+		assert.Equal(t, 120*time.Second, b.Backoff(10))
+		assert.Equal(t, 120*time.Second, b.Backoff(11))
+	})
+
+	t.Run("Invalid", func(t *testing.T) {
+		b := &ArangoBackupSpecBackOff{
+			MinDelay:   util.NewInt(-1),
+			MaxDelay:   util.NewInt(-1),
+			Iterations: util.NewInt(0),
+		}
+
+		assert.Equal(t, 0, b.GetMinDelay())
+		assert.Equal(t, 0, b.GetMaxDelay())
+		assert.Equal(t, 1, b.GetIterations())
+
+		assert.Equal(t, 0*time.Second, b.Backoff(12345))
+	})
+
+	t.Run("Max < Min", func(t *testing.T) {
+		b := &ArangoBackupSpecBackOff{
+			MinDelay: util.NewInt(50),
+			MaxDelay: util.NewInt(20),
+		}
+
+		assert.Equal(t, 20, b.GetMinDelay())
+		assert.Equal(t, 20, b.GetMaxDelay())
+		assert.Equal(t, 5, b.GetIterations())
+	})
+}

pkg/apis/backup/v1/backup_state.go

@@ -17,8 +17,6 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
-// Author Adam Janikowski
-//
 
 package v1
 

pkg/apis/backup/v1/backup_status.go

@@ -17,8 +17,6 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
-// Author Adam Janikowski
-//
 
 package v1
 
@@ -31,8 +29,9 @@ import (
 // an ArangoBackup.
 type ArangoBackupStatus struct {
 	ArangoBackupState `json:",inline"`
-	Backup            *ArangoBackupDetails `json:"backup,omitempty"`
-	Available         bool                 `json:"available"`
+	Backup            *ArangoBackupDetails       `json:"backup,omitempty"`
+	Available         bool                       `json:"available"`
+	Backoff           *ArangoBackupStatusBackOff `json:"backoff,omitempty"`
 }
 
 func (a *ArangoBackupStatus) Equal(b *ArangoBackupStatus) bool {

pkg/apis/backup/v1/backup_status_backoff.go

@@ -0,0 +1,59 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+import (
+	"time"
+
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type ArangoBackupStatusBackOff struct {
+	Iterations int       `json:"iterations,omitempty"`
+	Next       meta.Time `json:"next,omitempty"`
+}
+
+func (a *ArangoBackupStatusBackOff) GetIterations() int {
+	if a == nil {
+		return 0
+	}
+
+	if a.Iterations < 0 {
+		return 0
+	}
+
+	return a.Iterations
+}
+
+func (a *ArangoBackupStatusBackOff) GetNext() meta.Time {
+	if a == nil {
+		return meta.Time{}
+	}
+
+	return a.Next
+}
+
+func (a *ArangoBackupStatusBackOff) Backoff(spec *ArangoBackupSpecBackOff) *ArangoBackupStatusBackOff {
+	return &ArangoBackupStatusBackOff{
+		Iterations: a.GetIterations() + 1,
+		Next:       meta.Time{Time: time.Now().Add(spec.Backoff(a.GetIterations()))},
+	}
+}

pkg/apis/backup/v1/backup_status_backoff_test.go

@@ -0,0 +1,40 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2021 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestArangoBackupStatusBackOff_Backoff(t *testing.T) {
+	t.Run("Default", func(t *testing.T) {
+		var spec *ArangoBackupSpecBackOff
+		var status *ArangoBackupStatusBackOff
+
+		n := status.Backoff(spec)
+
+		require.Equal(t, 1, n.GetIterations())
+		require.True(t, n.GetNext().After(time.Now().Add(time.Duration(9.9*float64(time.Second)))))
+	})
+}

pkg/apis/backup/v1/backup_validate.go

@@ -17,8 +17,6 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
-// Author Adam Janikowski
-//
 
 package v1
 

pkg/apis/backup/v1/doc.go

@@ -17,8 +17,6 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
-// Author Adam Janikowski
-//
 
 // +k8s:deepcopy-gen=package
 // +groupName=backup.arangodb.com

pkg/apis/backup/v1/register.go

@@ -17,8 +17,6 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
-// Author Adam Janikowski
-//
 
 package v1
 

pkg/apis/backup/v1/zz_generated.deepcopy.go

@@ -273,6 +273,11 @@ func (in *ArangoBackupSpec) DeepCopyInto(out *ArangoBackupSpec) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.Backoff != nil {
+		in, out := &in.Backoff, &out.Backoff
+		*out = new(ArangoBackupSpecBackOff)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -286,6 +291,37 @@ func (in *ArangoBackupSpec) DeepCopy() *ArangoBackupSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ArangoBackupSpecBackOff) DeepCopyInto(out *ArangoBackupSpecBackOff) {
+	*out = *in
+	if in.MinDelay != nil {
+		in, out := &in.MinDelay, &out.MinDelay
+		*out = new(int)
+		**out = **in
+	}
+	if in.MaxDelay != nil {
+		in, out := &in.MaxDelay, &out.MaxDelay
+		*out = new(int)
+		**out = **in
+	}
+	if in.Iterations != nil {
+		in, out := &in.Iterations, &out.Iterations
+		*out = new(int)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoBackupSpecBackOff.
+func (in *ArangoBackupSpecBackOff) DeepCopy() *ArangoBackupSpecBackOff {
+	if in == nil {
+		return nil
+	}
+	out := new(ArangoBackupSpecBackOff)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ArangoBackupSpecDeployment) DeepCopyInto(out *ArangoBackupSpecDeployment) {
 	*out = *in
@@ -392,6 +428,11 @@ func (in *ArangoBackupStatus) DeepCopyInto(out *ArangoBackupStatus) {
 		*out = new(ArangoBackupDetails)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Backoff != nil {
+		in, out := &in.Backoff, &out.Backoff
+		*out = new(ArangoBackupStatusBackOff)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -405,6 +446,23 @@ func (in *ArangoBackupStatus) DeepCopy() *ArangoBackupStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ArangoBackupStatusBackOff) DeepCopyInto(out *ArangoBackupStatusBackOff) {
+	*out = *in
+	in.Next.DeepCopyInto(&out.Next)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoBackupStatusBackOff.
+func (in *ArangoBackupStatusBackOff) DeepCopy() *ArangoBackupStatusBackOff {
+	if in == nil {
+		return nil
+	}
+	out := new(ArangoBackupStatusBackOff)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ArangoBackupTemplate) DeepCopyInto(out *ArangoBackupTemplate) {
 	*out = *in

pkg/backup/handlers/arango/backup/state_upload.go

@@ -62,6 +62,7 @@ func stateUploadHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.
 			cleanStatusJob(),
 			updateStatusBackupUpload(nil),
 			updateStatusAvailable(true),
+			addBackOff(backup.Spec),
 		)
 	}
 

pkg/backup/handlers/arango/backup/state_upload_test.go

@@ -178,6 +178,9 @@ func Test_State_Upload_TemporaryUploadFailed(t *testing.T) {
 	// Assert
 	newObj := refreshArangoBackup(t, handler, obj)
 	checkBackup(t, newObj, backupApi.ArangoBackupStateUploadError, true)
+
+	require.NotNil(t, newObj.Status.Backoff)
+	require.Equal(t, 1, newObj.Status.Backoff.Iterations)
 }
 
 func Test_State_Upload_FatalUploadFailed(t *testing.T) {
@@ -205,4 +208,73 @@ func Test_State_Upload_FatalUploadFailed(t *testing.T) {
 	// Assert
 	newObj := refreshArangoBackup(t, handler, obj)
 	checkBackup(t, newObj, backupApi.ArangoBackupStateUploadError, true)
+
+	require.NotNil(t, newObj.Status.Backoff)
+	require.Equal(t, 1, newObj.Status.Backoff.Iterations)
+}
+
+func Test_State_Upload_TemporaryUploadFailed_Backoff(t *testing.T) {
+	// Arrange
+	error := newTemporaryErrorf("error")
+	handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{
+		uploadError: error,
+	})
+
+	obj, deployment := newObjectSet(backupApi.ArangoBackupStateUpload)
+
+	createResponse, err := mock.Create()
+	require.NoError(t, err)
+
+	obj.Status.Backup = createBackupFromMeta(driver.BackupMeta{
+		ID: createResponse.ID,
+	}, nil)
+	obj.Status.Backoff = &backupApi.ArangoBackupStatusBackOff{
+		Iterations: 3,
+	}
+
+	// Act
+	createArangoDeployment(t, handler, deployment)
+	createArangoBackup(t, handler, obj)
+
+	require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj)))
+
+	// Assert
+	newObj := refreshArangoBackup(t, handler, obj)
+	checkBackup(t, newObj, backupApi.ArangoBackupStateUploadError, true)
+
+	require.NotNil(t, newObj.Status.Backoff)
+	require.Equal(t, 4, newObj.Status.Backoff.Iterations)
+}
+
+func Test_State_Upload_FatalUploadFailed_Backoff(t *testing.T) {
+	// Arrange
+	error := newFatalErrorf("error")
+	handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{
+		uploadError: error,
+	})
+
+	obj, deployment := newObjectSet(backupApi.ArangoBackupStateUpload)
+
+	createResponse, err := mock.Create()
+	require.NoError(t, err)
+
+	obj.Status.Backup = createBackupFromMeta(driver.BackupMeta{
+		ID: createResponse.ID,
+	}, nil)
+	obj.Status.Backoff = &backupApi.ArangoBackupStatusBackOff{
+		Iterations: 3,
+	}
+
+	// Act
+	createArangoDeployment(t, handler, deployment)
+	createArangoBackup(t, handler, obj)
+
+	require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj)))
+
+	// Assert
+	newObj := refreshArangoBackup(t, handler, obj)
+	checkBackup(t, newObj, backupApi.ArangoBackupStateUploadError, true)
+
+	require.NotNil(t, newObj.Status.Backoff)
+	require.Equal(t, 4, newObj.Status.Backoff.Iterations)
 }

pkg/backup/handlers/arango/backup/state_uploaderror.go

@@ -28,12 +28,8 @@ import (
 	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
 )
 
-const (
-	uploadDelay = time.Minute
-)
-
 func stateUploadErrorHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) {
-	if backup.Spec.Upload == nil || backup.Status.Time.Time.Add(uploadDelay).Before(time.Now()) {
+	if backup.Spec.Upload == nil || !backup.Status.Backoff.GetNext().After(time.Now()) {
 		return wrapUpdateStatus(backup,
 			updateStatusState(backupApi.ArangoBackupStateReady, ""),
 			cleanStatusJob(),

pkg/backup/handlers/arango/backup/state_uploaderror_test.go

@@ -92,8 +92,10 @@ func Test_State_UploadError_Wait(t *testing.T) {
 		CreationTimestamp: meta.Now(),
 		Uploaded:          util.NewBool(true),
 	}
+	obj.Status.Backoff = &backupApi.ArangoBackupStatusBackOff{
+		Next: meta.Time{Time: time.Now().Add(5 * time.Second)},
+	}
 
-	obj.Status.Time.Time = time.Now().Add(2 * downloadDelay)
 	obj.Status.Message = "message"
 
 	// Act

pkg/backup/handlers/arango/backup/state_uploading.go

@@ -69,6 +69,7 @@ func stateUploadingHandler(h *handler, backup *backupApi.ArangoBackup) (*backupA
 				"Upload failed with error: %s", details.FailMessage),
 			cleanStatusJob(),
 			updateStatusAvailable(true),
+			addBackOff(backup.Spec),
 		)
 	}
 
@@ -78,6 +79,7 @@ func stateUploadingHandler(h *handler, backup *backupApi.ArangoBackup) (*backupA
 			cleanStatusJob(),
 			updateStatusBackupUpload(util.NewBool(true)),
 			updateStatusAvailable(true),
+			cleanBackOff(),
 		)
 	}
 

pkg/backup/handlers/arango/backup/status.go

@@ -67,6 +67,18 @@ func updateStatusAvailable(available bool) updateStatusFunc {
 	}
 }
 
+func cleanBackOff() updateStatusFunc {
+	return func(status *backupApi.ArangoBackupStatus) {
+		status.Backoff = nil
+	}
+}
+
+func addBackOff(spec backupApi.ArangoBackupSpec) updateStatusFunc {
+	return func(status *backupApi.ArangoBackupStatus) {
+		status.Backoff = status.Backoff.Backoff(spec.Backoff)
+	}
+}
+
 func updateStatusJob(id, progress string) updateStatusFunc {
 	return func(status *backupApi.ArangoBackupStatus) {
 		status.Progress = &backupApi.ArangoBackupProgress{