[Feature] Imported ArangoBackup Cleanup (#1667)

CHANGELOG.md

@@ -17,6 +17,7 @@
 - (Feature) (ML) Unify Integration Sidecar
 - (Feature) (Analytics) Metadata
 - (Feature) (Analytics) StatefulSet
+- (Feature) Imported ArangoBackup Cleanup
 
 ## [1.2.40](https://github.com/arangodb/kube-arangodb/tree/1.2.40) (2024-04-10)
 - (Feature) Add Core fields to the Scheduler Container Spec

README.md

@@ -73,9 +73,10 @@ covers individual newer features separately.
 [START_INJECT]: # (featuresCommunityTable)
 
 | Feature                                                                       | Operator Version | Introduced | ArangoDB Version | ArangoDB Edition      | State        | Enabled | Flag                                                   | Remarks                                                                                   |
-|:------------------------------------------------------------------------------|:-----------------|:-----------|:-----------------|:----------------------|:-------------|:--------|:-------------------------------------------------------|:--------------------------------------------------------------------------------------|
+|:------------------------------------------------------------------------------|:-----------------|:-----------|:-----------------|:----------------------|:-------------|:--------|:-------------------------------------------------------|:------------------------------------------------------------------------------------------|
+| Cleanup Imported Backups                                                      | 1.2.41           | 1.2.41     | >= 3.8.0         | Community, Enterprise | Production   | False   | --deployment.feature.backup-cleanup                    | Cleanup backups created outside of the Operator and imported into Kubernetes ArangoBackup |
 | Upscale resources spec in init containers                                     | 1.2.36           | 1.2.36     | >= 3.8.0         | Community, Enterprise | Production   | True    | --deployment.feature.init-containers-upscale-resources | Upscale resources spec to built-in init containers if they are not specified or lower     |
-| Create backups asynchronously                                                 | 1.2.35           | 1.2.35     | >= 3.8.0         | Community, Enterprise | Production   | False   | --deployment.feature.async-backup-creation             | Create backups asynchronously to avoid blocking the operator and reaching the timeout |
+| Create backups asynchronously                                                 | 1.2.35           | 1.2.41     | >= 3.8.0         | Community, Enterprise | Production   | True    | --deployment.feature.async-backup-creation             | Create backups asynchronously to avoid blocking the operator and reaching the timeout     |
 | Enforced ResignLeadership                                                     | 1.2.34           | 1.2.34     | >= 3.8.0         | Community, Enterprise | Production   | True    | --deployment.feature.enforced-resign-leadership        | Enforce ResignLeadership and ensure that Leaders are moved from restarted DBServer        |
 | Copy resources spec to init containers                                        | 1.2.33           | 1.2.33     | >= 3.8.0         | Community, Enterprise | Production   | True    | --deployment.feature.init-containers-copy-resources    | Copy resources spec to built-in init containers if they are not specified                 |
 | [Rebalancer V2](docs/features/rebalancer_v2.md)                               | 1.2.31           | 1.2.31     | >= 3.10.0        | Community, Enterprise | Alpha        | False   | --deployment.feature.rebalancer-v2                     | N/A                                                                                       |
@@ -122,6 +123,9 @@ To upgrade to the Enterprise Edition, you need to get in touch with the ArangoDB
 
 ## Flags
 
+[START_INJECT]: # (operatorArguments)
+
+
 ```
 Flags:
       --action.PVCResize.concurrency int                       Define limit of concurrent PVC Resizes on the cluster (default 32)
@@ -140,7 +144,8 @@ Flags:
       --crd.validation-schema stringArray                      Overrides default set of CRDs which should have validation schema enabled <crd-name>=<true/false>.
       --deployment.feature.agency-poll                         Enable Agency Poll for Enterprise deployments - Required ArangoDB 3.8.0 or higher (default true)
       --deployment.feature.all                                 Enable ALL Features
-      --deployment.feature.async-backup-creation               Create backups asynchronously to avoid blocking the operator and reaching the timeout - Required ArangoDB 3.8.0 or higher
+      --deployment.feature.async-backup-creation               Create backups asynchronously to avoid blocking the operator and reaching the timeout - Required ArangoDB 3.8.0 or higher (default true)
+      --deployment.feature.backup-cleanup                      Cleanup imported backups if required - Required ArangoDB 3.8.0 or higher
       --deployment.feature.deployment-spec-defaults-restore    Restore defaults from last accepted state of deployment - Required ArangoDB 3.8.0 or higher (default true)
       --deployment.feature.enforced-resign-leadership          Enforce ResignLeadership and ensure that Leaders are moved from restarted DBServer - Required ArangoDB 3.8.0 or higher (default true)
       --deployment.feature.ephemeral-volumes                   Enables ephemeral volumes for apps and tmp directory - Required ArangoDB 3.8.0 or higher
@@ -160,7 +165,6 @@ Flags:
       --deployment.feature.upgrade-version-check               Enable initContainer with pre version check - Required ArangoDB 3.8.0 or higher (default true)
       --deployment.feature.upgrade-version-check-v2            Enable initContainer with pre version check based by Operator - Required ArangoDB 3.8.0 or higher
       --features-config-map-name string                        Name of the Feature Map ConfigMap (default "arangodb-operator-feature-config-map")
-  -h, --help                                                   help for arangodb_operator
       --image.discovery.status                                 Discover Operator Image from Pod Status by default. When disabled Pod Spec is used. (default true)
       --image.discovery.timeout duration                       Timeout for image discovery process (default 1m0s)
       --internal.scaling-integration                           Enable Scaling Integration
@@ -168,11 +172,12 @@ Flags:
       --kubernetes.max-batch-size int                          Size of batch during objects read (default 256)
       --kubernetes.qps float32                                 Number of queries per second for k8s API (default 15)
       --log.format string                                      Set log format. Allowed values: 'pretty', 'JSON'. If empty, default format is used (default "pretty")
-      --log.level stringArray                                  Set log levels in format <level> or <logger>=<level>. Possible loggers: action, agency, api-server, assertion, backup-operator, chaos-monkey, crd, deployment, deployment-ci, deployment-reconcile, deployment-replication, deployment-resilience, deployment-resources, deployment-storage, deployment-storage-pc, deployment-storage-service, http, inspector, integrations, k8s-client, ml-batchjob-operator, ml-cronjob-operator, ml-extension-operator, ml-extension-shutdown, ml-storage-operator, monitor, operator, operator-arangojob-handler, operator-v2, operator-v2-event, operator-v2-worker, panics, pod_compare, root, root-event-recorder, scheduler, server, server-authentication (default [info])
+      --log.level stringArray                                  Set log levels in format <level> or <logger>=<level>. Possible loggers: action, agency, api-server, assertion, backup-operator, chaos-monkey, crd, deployment, deployment-ci, deployment-reconcile, deployment-replication, deployment-resilience, deployment-resources, deployment-storage, deployment-storage-pc, deployment-storage-service, http, inspector, integrations, k8s-client, monitor, operator, operator-arangojob-handler, operator-v2, operator-v2-event, operator-v2-worker, panics, pod_compare, root, root-event-recorder, server, server-authentication (default [info])
       --log.sampling                                           If true, operator will try to minimize duplication of logging events (default true)
       --memory-limit uint                                      Define memory limit for hard shutdown and the dump of goroutines. Used for testing
       --metrics.excluded-prefixes stringArray                  List of the excluded metrics prefixes
       --mode.single                                            Enable single mode in Operator. WARNING: There should be only one replica of Operator, otherwise Operator can take unexpected actions
+      --operator.analytics                                     Enable to run the Analytics operator
       --operator.apps                                          Enable to run the ArangoApps operator
       --operator.backup                                        Enable to run the ArangoBackup operator
       --operator.deployment                                    Enable to run the ArangoDeployment operator
@@ -204,7 +209,9 @@ Flags:
       --timeout.shard-rebuild-retry duration                   Timeout after which rebuild shards retry flow is triggered (default 4h0m0s)
 ```
 
-## Installation and Usage
+[END_INJECT]: # (operatorArguments)
+
+### Installation and Usage
 
 Docker images:
 - Community Edition: `arangodb/kube-arangodb:1.2.40`

cmd/cmd.go

@@ -262,6 +262,10 @@ func init() {
 	}
 }
 
+func Command() *cobra.Command {
+	return &cmdMain
+}
+
 func Execute() int {
 	flag.CommandLine.AddGoFlagSet(goflag.CommandLine)
 

docs/features/README.md

@@ -7,9 +7,10 @@ title: List of all features
 ## List of Community Edition features
 
 | Feature                                                         | Operator Version | Introduced | ArangoDB Version | ArangoDB Edition      | State        | Enabled | Flag                                                   | Remarks                                                                                   |
-|:----------------------------------------------------------------|:-----------------|:-----------|:-----------------|:----------------------|:-------------|:--------|:-------------------------------------------------------|:--------------------------------------------------------------------------------------|
+|:----------------------------------------------------------------|:-----------------|:-----------|:-----------------|:----------------------|:-------------|:--------|:-------------------------------------------------------|:------------------------------------------------------------------------------------------|
+| Cleanup Imported Backups                                        | 1.2.41           | 1.2.41     | >= 3.8.0         | Community, Enterprise | Production   | False   | --deployment.feature.backup-cleanup                    | Cleanup backups created outside of the Operator and imported into Kubernetes ArangoBackup |
 | Upscale resources spec in init containers                       | 1.2.36           | 1.2.36     | >= 3.8.0         | Community, Enterprise | Production   | True    | --deployment.feature.init-containers-upscale-resources | Upscale resources spec to built-in init containers if they are not specified or lower     |
-| Create backups asynchronously                                   | 1.2.35           | 1.2.35     | >= 3.8.0         | Community, Enterprise | Production   | False   | --deployment.feature.async-backup-creation             | Create backups asynchronously to avoid blocking the operator and reaching the timeout |
+| Create backups asynchronously                                   | 1.2.35           | 1.2.41     | >= 3.8.0         | Community, Enterprise | Production   | True    | --deployment.feature.async-backup-creation             | Create backups asynchronously to avoid blocking the operator and reaching the timeout     |
 | Enforced ResignLeadership                                       | 1.2.34           | 1.2.34     | >= 3.8.0         | Community, Enterprise | Production   | True    | --deployment.feature.enforced-resign-leadership        | Enforce ResignLeadership and ensure that Leaders are moved from restarted DBServer        |
 | Copy resources spec to init containers                          | 1.2.33           | 1.2.33     | >= 3.8.0         | Community, Enterprise | Production   | True    | --deployment.feature.init-containers-copy-resources    | Copy resources spec to built-in init containers if they are not specified                 |
 | [Rebalancer V2](rebalancer_v2.md)                               | 1.2.31           | 1.2.31     | >= 3.10.0        | Community, Enterprise | Alpha        | False   | --deployment.feature.rebalancer-v2                     | N/A                                                                                       |

internal/features.yaml

@@ -235,12 +235,21 @@ features:
       - operatorVersion: 1.2.34
         state: Production
   - name: Create backups asynchronously
-    enabled: false
+    enabled: true
     remarks: Create backups asynchronously to avoid blocking the operator and reaching the timeout
     flag: --deployment.feature.async-backup-creation
     releases:
+      - operatorVersion: 1.2.41
+        state: Production
       - operatorVersion: 1.2.35
         state: Production
+  - name: Cleanup Imported Backups
+    enabled: false
+    remarks: Cleanup backups created outside of the Operator and imported into Kubernetes ArangoBackup
+    flag: --deployment.feature.backup-cleanup
+    releases:
+      - operatorVersion: 1.2.41
+        state: Production
   - name: ArangoML integration
     operatorEditions: Enterprise
     arangoDBEditions: Enterprise

internal/readme.go

@@ -21,6 +21,7 @@
 package internal
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"path"
@@ -28,10 +29,12 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v3"
 
 	"github.com/arangodb/go-driver"
 
+	"github.com/arangodb/kube-arangodb/cmd"
 	"github.com/arangodb/kube-arangodb/internal/md"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
 	"github.com/arangodb/kube-arangodb/pkg/util"
@@ -128,6 +131,12 @@ func GenerateReadme(root string) error {
 		readmeSections["limits"] = section
 	}
 
+	if section, err := GenerateHelp(cmd.Command()); err != nil {
+		return err
+	} else {
+		readmeSections["operatorArguments"] = section
+	}
+
 	if err := md.ReplaceSectionsInFile(path.Join(root, "README.md"), readmeSections); err != nil {
 		return err
 	}
@@ -135,6 +144,34 @@ func GenerateReadme(root string) error {
 	return nil
 }
 
+func GenerateHelp(cmd *cobra.Command) (string, error) {
+	var lines []string
+
+	lines = append(lines, "```", "Flags:")
+
+	buff := bytes.NewBuffer(nil)
+
+	cmd.SetOut(buff)
+
+	cmd.SetArgs([]string{"--help"})
+
+	if err := cmd.Execute(); err != nil {
+		return "", err
+	}
+
+	help := buff.String()
+
+	for _, line := range strings.Split(help, "\n") {
+		if strings.HasPrefix(line, "      --") {
+			lines = append(lines, line)
+		}
+	}
+
+	lines = append(lines, "```")
+
+	return md.WrapWithNewLines(md.WrapWithNewLines(strings.Join(lines, "\n"))), nil
+}
+
 func GenerateReadmeFeatures(root, basePath string, eeOnly bool) (string, error) {
 	feature := md.NewColumn("Feature", md.ColumnLeftAlign)
 	introduced := md.NewColumn("Introduced", md.ColumnLeftAlign)

pkg/deployment/features/backup.go

@@ -22,12 +22,20 @@ package features
 
 func init() {
 	registerFeature(asyncBackupCreation)
+	registerFeature(backupCleanup)
 }
 
 var asyncBackupCreation = &feature{
 	name:               "async-backup-creation",
 	description:        "Create backups asynchronously to avoid blocking the operator and reaching the timeout",
 	enterpriseRequired: false,
+	enabledByDefault:   true,
+}
+
+var backupCleanup = &feature{
+	name:               "backup-cleanup",
+	description:        "Cleanup imported backups if required",
+	enterpriseRequired: false,
 	enabledByDefault:   false,
 }
 
@@ -35,3 +43,8 @@ var asyncBackupCreation = &feature{
 func AsyncBackupCreation() Feature {
 	return asyncBackupCreation
 }
+
+// BackupCleanup returns mode for Imported backups cleanup.
+func BackupCleanup() Feature {
+	return backupCleanup
+}

pkg/handlers/backup/handler.go

@@ -37,6 +37,7 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/apis/backup"
 	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
 	database "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
 	arangoClientSet "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned"
 	"github.com/arangodb/kube-arangodb/pkg/logging"
 	operator "github.com/arangodb/kube-arangodb/pkg/operatorV2"
@@ -133,6 +134,35 @@ func (h *handler) refreshDeployment(deployment *database.ArangoDeployment) error
 		}
 	}
 
+	if err = h.cleanupImportedBackups(backups.Items); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (h *handler) cleanupImportedBackups(backups []backupApi.ArangoBackup) error {
+	if !features.BackupCleanup().Enabled() {
+		return nil
+	}
+	for _, backup := range backups {
+		if backup.GetDeletionTimestamp() != nil {
+			continue
+		}
+
+		if b := backup.Status.Backup; b == nil || !util.TypeOrDefault(b.Imported, false) {
+			continue
+		}
+
+		logger.Str("name", backup.GetName()).Str("namespace", backup.GetNamespace()).Info("Removing Imported ArangoBackup")
+
+		err := h.client.BackupV1().ArangoBackups(backup.GetNamespace()).Delete(context.Background(), backup.GetName(), meta.DeleteOptions{})
+		if err != nil {
+			return err
+		}
+
+	}
+
 	return nil
 }
 
@@ -153,11 +183,15 @@ func (h *handler) refreshDeploymentBackup(deployment *database.ArangoDeployment,
 		}
 	}
 
+	name := fmt.Sprintf("backup-%s", uuid.NewUUID())
+
+	logger.Str("id", string(backupMeta.ID)).Strs("namespace", deployment.GetNamespace()).Str("namespace", name).Info("Importing ArangoBackup from API")
+
 	// New backup found, need to recreate
 	backup := &backupApi.ArangoBackup{
 		ObjectMeta: meta.ObjectMeta{
-			Name:      fmt.Sprintf("backup-%s", uuid.NewUUID()),
+			Name:      name,
-			Namespace: deployment.Namespace,
+			Namespace: deployment.GetNamespace(),
 		},
 		Spec: backupApi.ArangoBackupSpec{
 			Deployment: backupApi.ArangoBackupSpecDeployment{

pkg/handlers/backup/handler_test.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -23,11 +23,18 @@ package backup
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 	apiErrors "k8s.io/apimachinery/pkg/api/errors"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/uuid"
+
+	"github.com/arangodb/go-driver"
 
 	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
+	database "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
 	"github.com/arangodb/kube-arangodb/pkg/operatorV2/operation"
 	"github.com/arangodb/kube-arangodb/pkg/util/tests"
 )
@@ -59,3 +66,67 @@ func Test_ObjectNotFound(t *testing.T) {
 		})
 	}
 }
+
+func resetFeature(f features.Feature) func() {
+	enabled := f.Enabled()
+
+	return func() {
+		*f.EnabledPointer() = enabled
+	}
+}
+
+func Test_Refresh_Cleanup(t *testing.T) {
+	defer resetFeature(features.BackupCleanup())()
+
+	// Arrange
+	handler, client := newErrorsFakeHandler(mockErrorsArangoClientBackup{})
+
+	id := driver.BackupID(uuid.NewUUID())
+	client.state.backups = map[driver.BackupID]driver.BackupMeta{
+		id: {
+			ID:                      id,
+			Version:                 "3.12.0",
+			DateTime:                time.Now().Add(-time.Hour),
+			NumberOfFiles:           123,
+			NumberOfDBServers:       3,
+			SizeInBytes:             123,
+			PotentiallyInconsistent: false,
+			Available:               true,
+			NumberOfPiecesPresent:   123,
+		},
+	}
+
+	arangoDeployment := tests.NewMetaObject[*database.ArangoDeployment](t, tests.FakeNamespace, "deployment")
+
+	t.Run("Discover", func(t *testing.T) {
+		require.NoError(t, handler.refreshDeployment(arangoDeployment))
+
+		backups, err := handler.client.BackupV1().ArangoBackups(tests.FakeNamespace).List(context.Background(), meta.ListOptions{})
+		require.NoError(t, err)
+		require.Len(t, backups.Items, 1)
+		require.NotNil(t, backups.Items[0].Status.Backup)
+		require.EqualValues(t, id, backups.Items[0].Status.Backup.ID)
+	})
+
+	t.Run("Without Cleanup Feature", func(t *testing.T) {
+		*features.BackupCleanup().EnabledPointer() = false
+
+		require.NoError(t, handler.refreshDeployment(arangoDeployment))
+
+		backups, err := handler.client.BackupV1().ArangoBackups(tests.FakeNamespace).List(context.Background(), meta.ListOptions{})
+		require.NoError(t, err)
+		require.Len(t, backups.Items, 1)
+		require.NotNil(t, backups.Items[0].Status.Backup)
+		require.EqualValues(t, id, backups.Items[0].Status.Backup.ID)
+	})
+
+	t.Run("With Cleanup Feature", func(t *testing.T) {
+		*features.BackupCleanup().EnabledPointer() = true
+
+		require.NoError(t, handler.refreshDeployment(arangoDeployment))
+
+		backups, err := handler.client.BackupV1().ArangoBackups(tests.FakeNamespace).List(context.Background(), meta.ListOptions{})
+		require.NoError(t, err)
+		require.Len(t, backups.Items, 0)
+	})
+}