diff --git a/CHANGELOG.md b/CHANGELOG.md index d7bfd6fe2..9570190d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Change Log ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A) +- (Feature) ArangoBackup create retries and MaxIterations limit ## [1.2.27](https://github.com/arangodb/kube-arangodb/tree/1.2.27) (2023-04-27) - (Feature) Add InSync Cache diff --git a/pkg/apis/backup/v1/backup_spec_backoff.go b/pkg/apis/backup/v1/backup_spec_backoff.go index ebbdd4600..fbd8ecdfb 100644 --- a/pkg/apis/backup/v1/backup_spec_backoff.go +++ b/pkg/apis/backup/v1/backup_spec_backoff.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -29,6 +29,8 @@ type ArangoBackupSpecBackOff struct { MaxDelay *int `json:"max_delay,omitempty"` // Iterations defines number of iterations before reaching MaxDelay. Default to 5 Iterations *int `json:"iterations,omitempty"` + // MaxIterations defines maximum number of iterations after backoff will be disabled. Default to nil (no limit) + MaxIterations *int `json:"max_iterations,omitempty"` } func (a *ArangoBackupSpecBackOff) GetMaxDelay() int { diff --git a/pkg/apis/backup/v1/backup_state.go b/pkg/apis/backup/v1/backup_state.go index d990ab4e7..ecfcf30a4 100644 --- a/pkg/apis/backup/v1/backup_state.go +++ b/pkg/apis/backup/v1/backup_state.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -34,6 +34,7 @@ const ( ArangoBackupStateDownloadError state.State = "DownloadError" ArangoBackupStateDownloading state.State = "Downloading" ArangoBackupStateCreate state.State = "Create" + ArangoBackupStateCreateError state.State = "CreateError" ArangoBackupStateUpload state.State = "Upload" ArangoBackupStateUploading state.State = "Uploading" ArangoBackupStateUploadError state.State = "UploadError" @@ -50,7 +51,8 @@ var ArangoBackupStateMap = state.Map{ ArangoBackupStateDownload: {ArangoBackupStateDownloading, ArangoBackupStateFailed, ArangoBackupStateDownloadError}, ArangoBackupStateDownloading: {ArangoBackupStateReady, ArangoBackupStateFailed, ArangoBackupStateDownloadError}, ArangoBackupStateDownloadError: {ArangoBackupStatePending, ArangoBackupStateFailed}, - ArangoBackupStateCreate: {ArangoBackupStateReady, ArangoBackupStateFailed}, + ArangoBackupStateCreate: {ArangoBackupStateReady, ArangoBackupStateFailed, ArangoBackupStateCreateError}, + ArangoBackupStateCreateError: {ArangoBackupStateFailed, ArangoBackupStateCreate}, ArangoBackupStateUpload: {ArangoBackupStateUploading, ArangoBackupStateFailed, ArangoBackupStateDeleted, ArangoBackupStateUploadError}, ArangoBackupStateUploading: {ArangoBackupStateReady, ArangoBackupStateFailed, ArangoBackupStateUploadError}, ArangoBackupStateUploadError: {ArangoBackupStateFailed, ArangoBackupStateReady}, diff --git a/pkg/apis/backup/v1/backup_status_backoff.go b/pkg/apis/backup/v1/backup_status_backoff.go index 96c77edd3..6e2df565f 100644 --- a/pkg/apis/backup/v1/backup_status_backoff.go +++ b/pkg/apis/backup/v1/backup_status_backoff.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -51,7 +51,19 @@ func (a *ArangoBackupStatusBackOff) GetNext() meta.Time { return a.Next } +func (a *ArangoBackupStatusBackOff) ShouldBackoff(spec *ArangoBackupSpecBackOff) bool { + return spec == nil || spec.MaxIterations == nil || a.GetIterations() < *spec.MaxIterations +} + func (a *ArangoBackupStatusBackOff) Backoff(spec *ArangoBackupSpecBackOff) *ArangoBackupStatusBackOff { + if !a.ShouldBackoff(spec) { + // Do not backoff anymore + return &ArangoBackupStatusBackOff{ + Iterations: a.GetIterations(), + Next: a.GetNext(), + } + } + return &ArangoBackupStatusBackOff{ Iterations: a.GetIterations() + 1, Next: meta.Time{Time: time.Now().Add(spec.Backoff(a.GetIterations()))}, diff --git a/pkg/apis/backup/v1/backup_status_backoff_test.go b/pkg/apis/backup/v1/backup_status_backoff_test.go index 8ca9a4b0f..9c8d13d77 100644 --- a/pkg/apis/backup/v1/backup_status_backoff_test.go +++ b/pkg/apis/backup/v1/backup_status_backoff_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -25,6 +25,8 @@ import ( "time" "github.com/stretchr/testify/require" + + "github.com/arangodb/kube-arangodb/pkg/util" ) func TestArangoBackupStatusBackOff_Backoff(t *testing.T) { @@ -37,4 +39,21 @@ func TestArangoBackupStatusBackOff_Backoff(t *testing.T) { require.Equal(t, 1, n.GetIterations()) require.True(t, n.GetNext().After(time.Now().Add(time.Duration(9.9*float64(time.Second))))) }) + + t.Run("Test MaxIterations", func(t *testing.T) { + var spec = &ArangoBackupSpecBackOff{ + Iterations: util.NewInt(2), + MaxIterations: util.NewInt(3), + } + var status *ArangoBackupStatusBackOff + + n := status.Backoff(spec) + require.Equal(t, 1, n.GetIterations()) + require.True(t, n.ShouldBackoff(spec)) + + n.Iterations = 3 + n = n.Backoff(spec) + require.Equal(t, 3, n.GetIterations()) + require.False(t, n.ShouldBackoff(spec)) + }) } diff --git a/pkg/apis/backup/v1/zz_generated.deepcopy.go b/pkg/apis/backup/v1/zz_generated.deepcopy.go index adc315c1a..a34052661 100644 --- a/pkg/apis/backup/v1/zz_generated.deepcopy.go +++ b/pkg/apis/backup/v1/zz_generated.deepcopy.go @@ -310,6 +310,11 @@ func (in *ArangoBackupSpecBackOff) DeepCopyInto(out *ArangoBackupSpecBackOff) { *out = new(int) **out = **in } + if in.MaxIterations != nil { + in, out := &in.MaxIterations, &out.MaxIterations + *out = new(int) + **out = **in + } return } diff --git a/pkg/handlers/backup/state.go b/pkg/handlers/backup/state.go index 6315b5010..979c254d3 100644 --- a/pkg/handlers/backup/state.go +++ b/pkg/handlers/backup/state.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -33,6 +33,7 @@ var ( backupApi.ArangoBackupStatePending: statePendingHandler, backupApi.ArangoBackupStateScheduled: stateScheduledHandler, backupApi.ArangoBackupStateCreate: stateCreateHandler, + backupApi.ArangoBackupStateCreateError: stateCreateErrorHandler, backupApi.ArangoBackupStateUpload: stateUploadHandler, backupApi.ArangoBackupStateUploading: stateUploadingHandler, backupApi.ArangoBackupStateUploadError: stateUploadErrorHandler, diff --git a/pkg/handlers/backup/state_create.go b/pkg/handlers/backup/state_create.go index 709b275c4..b75d2136c 100644 --- a/pkg/handlers/backup/state_create.go +++ b/pkg/handlers/backup/state_create.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,6 +21,8 @@ package backup import ( + "time" + "github.com/arangodb/go-driver" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1" @@ -39,7 +41,12 @@ func stateCreateHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi. response, err := client.Create() if err != nil { - return nil, err + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateCreateError, "Create failed with error: %s", err.Error()), + cleanStatusJob(), + updateStatusAvailable(false), + addBackOff(backup.Spec), + ) } backupMeta, err := client.Get(response.ID) @@ -59,5 +66,25 @@ func stateCreateHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi. updateStatusState(backupApi.ArangoBackupStateReady, ""), updateStatusAvailable(true), updateStatusBackup(backupMeta), + cleanBackOff(), ) } + +func stateCreateErrorHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { + // no more retries - move to failed state + if !backup.Status.Backoff.ShouldBackoff(backup.Spec.Backoff) { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateFailed, "out of Create retries"), + cleanStatusJob()) + } + + // if we should retry - move to create state + if backup.Status.Backoff.ShouldBackoff(backup.Spec.Backoff) && !backup.Status.Backoff.GetNext().After(time.Now()) { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateCreate, ""), + cleanStatusJob()) + } + + // no ready to retry - wait (do not change state) + return wrapUpdateStatus(backup) +} diff --git a/pkg/handlers/backup/state_create_test.go b/pkg/handlers/backup/state_create_test.go index d7108bec3..b6d9eafd5 100644 --- a/pkg/handlers/backup/state_create_test.go +++ b/pkg/handlers/backup/state_create_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,8 +22,10 @@ package backup import ( "testing" + "time" "github.com/stretchr/testify/require" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/arangodb/go-driver" @@ -120,7 +122,7 @@ func Test_State_Create_Upload(t *testing.T) { compareBackupMeta(t, backupMeta, newObj) } -func Test_State_Create_CreateFailed(t *testing.T) { +func Test_State_Create_CreateError(t *testing.T) { // Arrange error := newFatalErrorf("error") handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{ @@ -137,32 +139,75 @@ func Test_State_Create_CreateFailed(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateFailed) - require.Equal(t, newObj.Status.Message, createStateMessage(backupApi.ArangoBackupStateCreate, backupApi.ArangoBackupStateFailed, error.Error())) - + require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateCreateError) require.Nil(t, newObj.Status.Backup) - require.False(t, newObj.Status.Available) } -func Test_State_Create_TemporaryCreateFailed(t *testing.T) { +func Test_State_CreateError_Retry(t *testing.T) { // Arrange - error := newTemporaryErrorf("error") - handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{ - createError: error, - }) + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{}) - obj, deployment := newObjectSet(backupApi.ArangoBackupStateCreate) + obj, deployment := newObjectSet(backupApi.ArangoBackupStateCreateError) + + backupMeta, err := mock.Create() + require.NoError(t, err) + + obj.Status.Backup = &backupApi.ArangoBackupDetails{ + ID: string(backupMeta.ID), + Version: backupMeta.Version, + CreationTimestamp: meta.Now(), + } + + obj.Status.Time.Time = time.Now().Add(-2 * downloadDelay) // Act createArangoDeployment(t, handler, deployment) createArangoBackup(t, handler, obj) - err := handler.Handle(newItemFromBackup(operation.Update, obj)) - require.EqualError(t, err, error.Error()) + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) // Assert newObj := refreshArangoBackup(t, handler, obj) - - require.Equal(t, obj.Status, newObj.Status) + require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateCreate) + require.False(t, newObj.Status.Available) + require.NotNil(t, newObj.Status.Backup) + require.Equal(t, obj.Status.Backup, newObj.Status.Backup) +} + +func Test_State_CreateError_Transfer_To_Failed(t *testing.T) { + // Arrange + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{}) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateCreateError) + + backupMeta, err := mock.Create() + require.NoError(t, err) + + obj.Status.Backup = &backupApi.ArangoBackupDetails{ + ID: string(backupMeta.ID), + Version: backupMeta.Version, + CreationTimestamp: meta.Now(), + } + obj.Status.Backoff = &backupApi.ArangoBackupStatusBackOff{ + Iterations: 2, + } + + obj.Spec.Backoff = &backupApi.ArangoBackupSpecBackOff{ + Iterations: util.NewInt(1), + MaxIterations: util.NewInt(2), + } + + obj.Status.Time.Time = time.Now().Add(-2 * downloadDelay) + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateFailed) + require.Equal(t, newObj.Status.Message, "out of Create retries") } diff --git a/pkg/handlers/backup/state_uploaderror.go b/pkg/handlers/backup/state_uploaderror.go index a1deaa635..abfa4edc5 100644 --- a/pkg/handlers/backup/state_uploaderror.go +++ b/pkg/handlers/backup/state_uploaderror.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -27,13 +27,23 @@ import ( ) func stateUploadErrorHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { - if backup.Spec.Upload == nil || !backup.Status.Backoff.GetNext().After(time.Now()) { + // no more retries - move to failed state + if !backup.Status.Backoff.ShouldBackoff(backup.Spec.Backoff) { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateFailed, "out of Upload retries"), + cleanStatusJob()) + } + + // if we should retry - move to ready state + if backup.Spec.Upload == nil || + (backup.Status.Backoff.ShouldBackoff(backup.Spec.Backoff) && !backup.Status.Backoff.GetNext().After(time.Now())) { return wrapUpdateStatus(backup, updateStatusState(backupApi.ArangoBackupStateReady, ""), cleanStatusJob(), updateStatusAvailable(true)) } + // no ready to retry - wait (do not change state) return wrapUpdateStatus(backup, updateStatusAvailable(true)) }