[Feature] Restore as Plan (#560)

CHANGELOG.md

@@ -1,7 +1,8 @@
 # Change Log
 
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
-- Prevent deletion not known PVC's
+- Prevent deletion of not known PVC's
+- Move Restore as Plan
 
 ## [1.0.2](https://github.com/arangodb/kube-arangodb/tree/1.0.2) (2020-04-16)
 - Added additional checks in UpToDate condition

pkg/apis/deployment/v1/plan.go

@@ -75,6 +75,10 @@ const (
 	ActionTypePVCResized ActionType = "PVCResized"
 	// UpToDateUpdateResized define up to date annotation in spec
 	UpToDateUpdate ActionType = "UpToDateUpdate"
+	// ActionTypeBackupRestore restore plan
+	ActionTypeBackupRestore ActionType = "BackupRestore"
+	// ActionTypeBackupRestoreClean restore plan
+	ActionTypeBackupRestoreClean ActionType = "BackupRestoreClean"
 )
 
 const (

pkg/deployment/backup/handler.go

@@ -1,134 +0,0 @@
-//
-// DISCLAIMER
-//
-// Copyright 2020 ArangoDB GmbH, Cologne, Germany
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Copyright holder is ArangoDB GmbH, Cologne, Germany
-//
-// Author Lars Maier
-//
-
-package backup
-
-import (
-	"context"
-
-	"github.com/arangodb/go-driver"
-	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
-	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
-	"github.com/rs/zerolog"
-)
-
-type Context interface {
-	// GetSpec returns the current specification of the deployment
-	GetSpec() api.DeploymentSpec
-	// GetStatus returns the current status of the deployment
-	GetStatus() (api.DeploymentStatus, int32)
-	// UpdateStatus replaces the status of the deployment with the given status and
-	// updates the resources in k8s.
-	UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error
-	// GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server),
-	// creating one if needed.
-	GetDatabaseClient(ctx context.Context) (driver.Client, error)
-	// GetBackup receives information about a backup resource
-	GetBackup(backup string) (*backupApi.ArangoBackup, error)
-}
-
-type BackupHandler struct {
-	log     zerolog.Logger
-	context Context
-}
-
-func NewHandler(log zerolog.Logger, context Context) *BackupHandler {
-	return &BackupHandler{
-		log:     log,
-		context: context,
-	}
-}
-
-func (b *BackupHandler) restoreFrom(backupName string) error {
-	ctx := context.Background()
-	dbc, err := b.context.GetDatabaseClient(ctx)
-	if err != nil {
-		return err
-	}
-
-	backupResource, err := b.context.GetBackup(backupName)
-	if err != nil {
-		return err
-	}
-
-	backupID := backupResource.Status.Backup.ID
-
-	// trigger the actual restore
-	if err := dbc.Backup().Restore(ctx, driver.BackupID(backupID), nil); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (b *BackupHandler) CheckRestore() error {
-
-	spec := b.context.GetSpec()
-	status, version := b.context.GetStatus()
-
-	if spec.HasRestoreFrom() {
-		// We have to trigger a restore operation
-		if status.Restore == nil || status.Restore.RequestedFrom != spec.GetRestoreFrom() {
-			// Prepare message that we are starting restore
-			result := &api.DeploymentRestoreResult{
-				RequestedFrom: spec.GetRestoreFrom(),
-			}
-
-			result.State = api.DeploymentRestoreStateRestoring
-
-			for i := 0; i < 100; i++ {
-				status, version := b.context.GetStatus()
-				status.Restore = result
-				b.context.UpdateStatus(status, version)
-			}
-
-			// Request restoring
-			err := b.restoreFrom(spec.GetRestoreFrom())
-
-			if err != nil {
-				result.State = api.DeploymentRestoreStateRestoreFailed
-				result.Message = err.Error()
-			} else {
-				result.State = api.DeploymentRestoreStateRestored
-			}
-
-			// try to update the status
-			for i := 0; i < 100; i++ {
-				status, version := b.context.GetStatus()
-				status.Restore = result
-				b.context.UpdateStatus(status, version)
-			}
-		}
-
-		return nil
-	}
-
-	if status.Restore == nil {
-		return nil
-	}
-
-	// Remove the restore entry from status
-	status.Restore = nil
-	b.context.UpdateStatus(status, version)
-	return nil
-
-}

pkg/deployment/deployment.go

@@ -39,7 +39,6 @@ import (
 	"k8s.io/client-go/tools/record"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
-	"github.com/arangodb/kube-arangodb/pkg/deployment/backup"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/chaos"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/reconcile"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resilience"
@@ -112,7 +111,6 @@ type Deployment struct {
 	reconciler                *reconcile.Reconciler
 	resilience                *resilience.Resilience
 	resources                 *resources.Resources
-	backup                    *backup.BackupHandler
 	chaosMonkey               *chaos.Monkey
 	syncClientCache           client.ClientCache
 	haveServiceMonitorCRD     bool
@@ -135,7 +133,6 @@ func New(config Config, deps Dependencies, apiObject *api.ArangoDeployment) (*De
 	d.reconciler = reconcile.NewReconciler(deps.Log, d)
 	d.resilience = resilience.NewResilience(deps.Log, d)
 	d.resources = resources.NewResources(deps.Log, d)
-	d.backup = backup.NewHandler(deps.Log, d)
 	if d.status.last.AcceptedSpec == nil {
 		// We've validated the spec, so let's use it from now.
 		d.status.last.AcceptedSpec = apiObject.Spec.DeepCopy()

pkg/deployment/deployment_inspector.go

@@ -236,10 +236,6 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva
 		return minInspectionInterval, errors.Wrapf(err, "Removed member cleanup failed")
 	}
 
-	if err := d.backup.CheckRestore(); err != nil {
-		return minInspectionInterval, errors.Wrapf(err, "Restore operation failed")
-	}
-
 	// At the end of the inspect, we cleanup terminated pods.
 	if x, err := d.resources.CleanupTerminatedPods(); err != nil {
 		return minInspectionInterval, errors.Wrapf(err, "Pod cleanup failed")

pkg/deployment/reconcile/action_backup_restore.go

@@ -0,0 +1,123 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	"github.com/arangodb/go-driver"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/rs/zerolog"
+)
+
+func init() {
+	registerAction(api.ActionTypeBackupRestore, newBackupRestoreAction)
+}
+
+func newBackupRestoreAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &actionBackupRestore{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, backupRestoreTimeout)
+
+	return a
+}
+
+// actionBackupRestore implements an BackupRestore.
+type actionBackupRestore struct {
+	// actionImpl implement timeout and member id functions
+	actionImpl
+
+	actionEmptyCheckProgress
+}
+
+func (a actionBackupRestore) Start(ctx context.Context) (bool, error) {
+	spec := a.actionCtx.GetSpec()
+	status := a.actionCtx.GetStatus()
+
+	if spec.RestoreFrom == nil {
+		return true, nil
+	}
+
+	if status.Restore != nil {
+		a.log.Warn().Msg("Backup restore status should not be nil")
+		return true, nil
+	}
+
+	dbc, err := a.actionCtx.GetDatabaseClient(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	backupResource, err := a.actionCtx.GetBackup(*spec.RestoreFrom)
+	if err != nil {
+		a.log.Error().Err(err).Msg("Unable to find backup")
+		return true, nil
+	}
+
+	if backupResource.Status.Backup == nil {
+		a.log.Error().Msg("Backup ID is not set")
+		return true, nil
+	}
+
+	if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool {
+		result := &api.DeploymentRestoreResult{
+			RequestedFrom: spec.GetRestoreFrom(),
+		}
+
+		result.State = api.DeploymentRestoreStateRestoring
+
+		s.Restore = result
+
+		return true
+	}, true); err != nil {
+		return false, err
+	}
+
+	restoreError := dbc.Backup().Restore(ctx, driver.BackupID(backupResource.Status.Backup.ID), nil)
+	if restoreError != nil {
+		a.log.Error().Err(restoreError).Msg("Restore failed")
+	}
+
+	if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool {
+		result := &api.DeploymentRestoreResult{
+			RequestedFrom: spec.GetRestoreFrom(),
+		}
+
+		if restoreError != nil {
+			result.State = api.DeploymentRestoreStateRestoreFailed
+			result.Message = restoreError.Error()
+		} else {
+			result.State = api.DeploymentRestoreStateRestored
+		}
+
+		s.Restore = result
+
+		return true
+	}); err != nil {
+		a.log.Error().Err(err).Msg("Unable to ser restored state")
+		return false, err
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/action_backup_restore_clean.go

@@ -0,0 +1,65 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/rs/zerolog"
+)
+
+func init() {
+	registerAction(api.ActionTypeBackupRestoreClean, newBackupRestoreCleanAction)
+}
+
+func newBackupRestoreCleanAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &actionBackupRestoreClean{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, backupRestoreTimeout)
+
+	return a
+}
+
+// actionBackupRestoreClean implements an BackupRestoreClean.
+type actionBackupRestoreClean struct {
+	// actionImpl implement timeout and member id functions
+	actionImpl
+
+	actionEmptyCheckProgress
+}
+
+func (a actionBackupRestoreClean) Start(ctx context.Context) (bool, error) {
+	if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool {
+		if s.Restore == nil {
+			return false
+		}
+
+		s.Restore = nil
+		return true
+	}, true); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/action_context.go

@@ -26,6 +26,8 @@ import (
 	"context"
 	"fmt"
 
+	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
+
 	"github.com/arangodb/go-driver/agency"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	v1 "k8s.io/api/core/v1"
@@ -110,6 +112,8 @@ type ActionContext interface {
 	InvalidateSyncStatus()
 	// GetSpec returns a copy of the spec
 	GetSpec() api.DeploymentSpec
+	// GetStatus returns a copy of the status
+	GetStatus() api.DeploymentStatus
 	// DisableScalingCluster disables scaling DBservers and coordinators
 	DisableScalingCluster() error
 	// EnableScalingCluster enables scaling DBservers and coordinators
@@ -117,6 +121,10 @@ type ActionContext interface {
 	// WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken
 	UpdateClusterCondition(conditionType api.ConditionType, status bool, reason, message string) error
 	SecretsInterface() k8sutil.SecretInterface
+	// WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken
+	WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error
+	// GetBackup receives information about a backup resource
+	GetBackup(backup string) (*backupApi.ArangoBackup, error)
 }
 
 // newActionContext creates a new ActionContext implementation.
@@ -133,6 +141,22 @@ type actionContext struct {
 	context Context
 }
 
+func (ac *actionContext) GetStatus() api.DeploymentStatus {
+	a, _ := ac.context.GetStatus()
+
+	s := a.DeepCopy()
+
+	return *s
+}
+
+func (ac *actionContext) GetBackup(backup string) (*backupApi.ArangoBackup, error) {
+	return ac.context.GetBackup(backup)
+}
+
+func (ac *actionContext) WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error {
+	return ac.context.WithStatusUpdate(action, force...)
+}
+
 func (ac *actionContext) SecretsInterface() k8sutil.SecretInterface {
 	return ac.context.SecretsInterface()
 }

pkg/deployment/reconcile/context.go

@@ -25,6 +25,8 @@ package reconcile
 import (
 	"context"
 
+	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
+
 	"github.com/arangodb/arangosync-client/client"
 	driver "github.com/arangodb/go-driver"
 	"github.com/arangodb/go-driver/agency"
@@ -115,4 +117,6 @@ type Context interface {
 	WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error
 	// SecretsInterface return secret interface
 	SecretsInterface() k8sutil.SecretInterface
+	// GetBackup receives information about a backup resource
+	GetBackup(backup string) (*backupApi.ArangoBackup, error)
 }

pkg/deployment/reconcile/plan_builder.go

@@ -232,6 +232,10 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
 		plan = createRotateTLSServerSNIPlan(ctx, log, spec, status, builderCtx)
 	}
 
+	if plan.IsEmpty() {
+		plan = createRestorePlan(ctx, log, spec, status, builderCtx)
+	}
+
 	// Return plan
 	return plan, true
 }

pkg/deployment/reconcile/plan_builder_context.go

@@ -25,6 +25,8 @@ package reconcile
 import (
 	"context"
 
+	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
+
 	"github.com/arangodb/go-driver"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
@@ -61,6 +63,8 @@ type PlanBuilderContext interface {
 	GetServerClient(ctx context.Context, group api.ServerGroup, id string) (driver.Client, error)
 	// SecretsInterface return secret interface
 	SecretsInterface() k8sutil.SecretInterface
+	// GetBackup receives information about a backup resource
+	GetBackup(backup string) (*backupApi.ArangoBackup, error)
 }
 
 // newPlanBuilderContext creates a PlanBuilderContext from the given context

pkg/deployment/reconcile/plan_builder_restore.go

@@ -0,0 +1,57 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Tomasz Mielech <tomasz@arangodb.com>
+//
+
+package reconcile
+
+import (
+	"context"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/rs/zerolog"
+)
+
+func createRestorePlan(ctx context.Context, log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus, builderCtx PlanBuilderContext) api.Plan {
+	if spec.RestoreFrom == nil && status.Restore != nil {
+		return api.Plan{
+			api.NewAction(api.ActionTypeBackupRestoreClean, api.ServerGroupUnknown, ""),
+		}
+	}
+
+	if spec.RestoreFrom != nil && status.Restore == nil {
+		backup, err := builderCtx.GetBackup(spec.GetRestoreFrom())
+		if err != nil {
+			log.Warn().Err(err).Msg("Backup not found")
+			return nil
+		}
+
+		if backup.Status.Backup == nil {
+			log.Warn().Msg("Backup not yet ready")
+			return nil
+		}
+
+		return api.Plan{
+			api.NewAction(api.ActionTypeBackupRestore, api.ServerGroupUnknown, ""),
+		}
+	}
+
+	return nil
+}

pkg/deployment/reconcile/plan_builder_test.go

@@ -29,6 +29,8 @@ import (
 	"io/ioutil"
 	"testing"
 
+	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
+
 	"github.com/arangodb/arangosync-client/client"
 	"github.com/arangodb/go-driver/agency"
 	"github.com/rs/zerolog"
@@ -55,6 +57,10 @@ type testContext struct {
 	RecordedEvent    *k8sutil.Event
 }
 
+func (c *testContext) GetBackup(backup string) (*backupApi.ArangoBackup, error) {
+	panic("implement me")
+}
+
 func (c *testContext) SecretsInterface() k8sutil.SecretInterface {
 	panic("implement me")
 }

pkg/deployment/reconcile/timeouts.go

@@ -34,6 +34,7 @@ const (
 	rotateMemberTimeout          = time.Minute * 15
 	pvcResizeTimeout             = time.Minute * 15
 	pvcResizedTimeout            = time.Minute * 15
+	backupRestoreTimeout         = time.Minute * 15
 	shutdownMemberTimeout        = time.Minute * 30
 	upgradeMemberTimeout         = time.Hour * 6
 	waitForMemberUpTimeout       = time.Minute * 15

pkg/deployment/resources/context.go

@@ -25,6 +25,8 @@ package resources
 import (
 	"context"
 
+	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
+
 	driver "github.com/arangodb/go-driver"
 	"github.com/arangodb/go-driver/agency"
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
@@ -94,4 +96,6 @@ type Context interface {
 	GetAgency(ctx context.Context) (agency.Agency, error)
 	// WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken
 	WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error
+	// GetBackup receives information about a backup resource
+	GetBackup(backup string) (*backupApi.ArangoBackup, error)
 }