[Feature] Add operator shutdown handler for graceful termination (#965)

CHANGELOG.md

@@ -7,6 +7,7 @@
 - (Feature) (ACS) Add Resource plan
 - (Feature) Allow raw json value for license token-v2
 - (Update) Replace `beta.kubernetes.io/arch` to `kubernetes.io/arch` in Operator Chart
+- (Feature) Add operator shutdown handler for graceful termination
 
 ## [1.2.12](https://github.com/arangodb/kube-arangodb/tree/1.2.12) (2022-05-10)
 - (Feature) Add CoreV1 Endpoints Inspector

cmd/main.go

@@ -33,9 +33,10 @@ import (
 
 	"github.com/arangodb/kube-arangodb/pkg/util/globals"
 
-	operatorHTTP "github.com/arangodb/kube-arangodb/pkg/util/http"
 	"github.com/gin-gonic/gin"
 
+	operatorHTTP "github.com/arangodb/kube-arangodb/pkg/util/http"
+
 	"github.com/arangodb/kube-arangodb/pkg/version"
 
 	"github.com/arangodb/kube-arangodb/pkg/operator/scope"
@@ -61,6 +62,9 @@ import (
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/record"
 
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/klog"
+
 	"github.com/arangodb/kube-arangodb/pkg/crd"
 	"github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/scheme"
 	"github.com/arangodb/kube-arangodb/pkg/logging"
@@ -71,8 +75,6 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util/kclient"
 	"github.com/arangodb/kube-arangodb/pkg/util/probe"
 	"github.com/arangodb/kube-arangodb/pkg/util/retry"
-	v1 "k8s.io/api/core/v1"
-	"k8s.io/klog"
 )
 
 const (
@@ -83,6 +85,8 @@ const (
 	defaultAlpineImage          = "alpine:3.7"
 	defaultMetricsExporterImage = "arangodb/arangodb-exporter:0.1.6"
 	defaultArangoImage          = "arangodb/arangodb:latest"
+	defaultShutdownDelay        = 2 * time.Second
+	defaultShutdownTimeout      = 30 * time.Second
 
 	UBIImageEnv             util.EnvironmentVariable = "RELATED_IMAGE_UBI"
 	ArangoImageEnv          util.EnvironmentVariable = "RELATED_IMAGE_DATABASE"
@@ -121,6 +125,10 @@ var (
 		singleMode bool
 		scope      string
 	}
+	shutdownOptions struct {
+		delay   time.Duration
+		timeout time.Duration
+	}
 	crdOptions struct {
 		install bool
 	}
@@ -180,6 +188,8 @@ func init() {
 	f.DurationVar(&operatorTimeouts.arangoDCheck, "timeout.arangod-check", globals.DefaultArangoDCheckTimeout, "The version check request timeout to the ArangoDB")
 	f.DurationVar(&operatorTimeouts.agency, "timeout.agency", globals.DefaultArangoDAgencyTimeout, "The Agency read timeout")
 	f.DurationVar(&operatorTimeouts.reconciliation, "timeout.reconciliation", globals.DefaultReconciliationTimeout, "The reconciliation timeout to the ArangoDB CR")
+	f.DurationVar(&shutdownOptions.delay, "shutdown.delay", defaultShutdownDelay, "The delay before running shutdown handlers")
+	f.DurationVar(&shutdownOptions.timeout, "shutdown.timeout", defaultShutdownTimeout, "Timeout for shutdown handlers")
 	f.BoolVar(&operatorOptions.scalingIntegrationEnabled, "internal.scaling-integration", true, "Enable Scaling Integration")
 	f.Int64Var(&operatorKubernetesOptions.maxBatchSize, "kubernetes.max-batch-size", globals.DefaultKubernetesRequestBatchSize, "Size of batch during objects read")
 	f.Float32Var(&operatorKubernetesOptions.qps, "kubernetes.qps", kclient.DefaultQPS, "Number of queries per second for k8s API")
@@ -421,6 +431,8 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper
 		ArangoImage:                 operatorOptions.arangoImage,
 		SingleMode:                  operatorOptions.singleMode,
 		Scope:                       scope,
+		ShutdownDelay:               shutdownOptions.delay,
+		ShutdownTimeout:             shutdownOptions.timeout,
 	}
 	deps := operator.Dependencies{
 		LogService:                 logService,

docs/design/README.md

@@ -9,4 +9,5 @@
 - [Status](./status.md)
 - [Upgrading](./upgrading.md)
 - [Rotating Pods](./rotating.md)
-- [Maintenance](./maintenance.md)
+- [Maintenance](./maintenance.md)
+- [Additional configuration](./additional_configuration.md)

docs/design/additional_configuration.md

@@ -0,0 +1,17 @@
+# Additional configuration
+
+It is possible to additionally fine-tune operator behavior by
+providing arguments via `operator.args` chart template value.
+
+For example, you can specify burst size for k8s API requests or how long the operator
+should wait for ArangoDeployment termination after receiving interruption signal:
+```
+operator:
+  args: ["--kubernetes.burst=40", --shutdown.timeout=2m"]
+```
+
+The full list of available arguments can be retrieved using 
+```
+export OPERATOR_IMAGE=arangodb/kube-arangodb:1.2.9
+kubectl run arango-operator-help --image=$OPERATOR_IMAGE -i --rm --restart=Never -- --help
+```

pkg/operator/operator.go

@@ -54,6 +54,7 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/operatorV2/event"
 	"github.com/arangodb/kube-arangodb/pkg/replication"
 	"github.com/arangodb/kube-arangodb/pkg/storage"
+	"github.com/arangodb/kube-arangodb/pkg/util"
 	"github.com/arangodb/kube-arangodb/pkg/util/constants"
 	"github.com/arangodb/kube-arangodb/pkg/util/kclient"
 	"github.com/arangodb/kube-arangodb/pkg/util/probe"
@@ -105,6 +106,8 @@ type Config struct {
 	ScalingIntegrationEnabled   bool
 	SingleMode                  bool
 	Scope                       scope.Scope
+	ShutdownDelay               time.Duration
+	ShutdownTimeout             time.Duration
 }
 
 type Dependencies struct {
@@ -177,8 +180,34 @@ func (o *Operator) Run() {
 			go o.runWithoutLeaderElection("arango-k2k-cluster-sync-operator", constants.ClusterSyncLabelRole, o.onStartK2KClusterSync, o.Dependencies.K2KClusterSyncProbe)
 		}
 	}
-	// Wait until process terminates
-	<-context.TODO().Done()
+
+	ctx := util.CreateSignalContext(context.Background())
+	<-ctx.Done()
+	o.log.Info().Msgf("Got interrupt signal, running shutdown handler in %s...", o.Config.ShutdownDelay)
+	time.Sleep(o.Config.ShutdownDelay)
+	o.handleShutdown()
+}
+
+func (o *Operator) handleShutdown() {
+	o.log.Info().Msg("Waiting for deployments termination...")
+	shutdownCh := make(chan struct{})
+	go func() {
+		for {
+			if len(o.deployments) == 0 {
+				break
+			}
+			time.Sleep(time.Second)
+		}
+		shutdownCh <- struct{}{}
+	}()
+	select {
+	case <-shutdownCh:
+		o.log.Info().Msg("All deployments terminated, exiting.")
+		return
+	case <-time.After(o.Config.ShutdownTimeout):
+		o.log.Info().Msg("Timeout reached before all deployments terminated, exiting.")
+		return
+	}
 }
 
 // onStartDeployment starts the deployment operator and run till given channel is closed.