Merge pull request #165 from arangodb/bugfix/deployment-finalizer

Added finalizer on deployment, used to remove child finalizers on delete

pkg/deployment/deployment.go

@@ -345,13 +345,17 @@ func (d *Deployment) updateCRStatus(force ...bool) error {
 	}
 
 	// Send update to API server
+	ns := d.apiObject.GetNamespace()
+	depls := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(ns)
 	update := d.apiObject.DeepCopy()
 	attempt := 0
 	for {
 		attempt++
 		update.Status = d.status
-		ns := d.apiObject.GetNamespace()
-		newAPIObject, err := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(ns).Update(update)
+		if update.GetDeletionTimestamp() == nil {
+			ensureFinalizers(update)
+		}
+		newAPIObject, err := depls.Update(update)
 		if err == nil {
 			// Update internal object
 			d.apiObject = newAPIObject
@@ -361,7 +365,7 @@ func (d *Deployment) updateCRStatus(force ...bool) error {
 			// API object may have been changed already,
 			// Reload api object and try again
 			var current *api.ArangoDeployment
-			current, err = d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(ns).Get(update.GetName(), metav1.GetOptions{})
+			current, err = depls.Get(update.GetName(), metav1.GetOptions{})
 			if err == nil {
 				update = current.DeepCopy()
 				continue

pkg/deployment/deployment_finalizers.go

@@ -0,0 +1,116 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package deployment
+
+import (
+	"context"
+
+	"github.com/rs/zerolog"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
+	"github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned"
+	"github.com/arangodb/kube-arangodb/pkg/util/constants"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+)
+
+// ensureFinalizers adds all required finalizers to the given deployment (in memory).
+func ensureFinalizers(depl *api.ArangoDeployment) {
+	for _, f := range depl.GetFinalizers() {
+		if f == constants.FinalizerDeplRemoveChildFinalizers {
+			// Finalizer already set
+			return
+		}
+	}
+	// Set finalizers
+	depl.SetFinalizers(append(depl.GetFinalizers(), constants.FinalizerDeplRemoveChildFinalizers))
+}
+
+// runDeploymentFinalizers goes through the list of ArangoDeployoment finalizers to see if they can be removed.
+func (d *Deployment) runDeploymentFinalizers(ctx context.Context) error {
+	log := d.deps.Log
+	var removalList []string
+
+	depls := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(d.GetNamespace())
+	updated, err := depls.Get(d.apiObject.GetName(), metav1.GetOptions{})
+	if err != nil {
+		return maskAny(err)
+	}
+	for _, f := range updated.ObjectMeta.GetFinalizers() {
+		switch f {
+		case constants.FinalizerDeplRemoveChildFinalizers:
+			log.Debug().Msg("Inspecting 'remove child finalizers' finalizer")
+			if err := d.inspectRemoveChildFinalizers(ctx, log, updated); err == nil {
+				removalList = append(removalList, f)
+			} else {
+				log.Debug().Err(err).Str("finalizer", f).Msg("Cannot remove finalizer yet")
+			}
+		}
+	}
+	// Remove finalizers (if needed)
+	if len(removalList) > 0 {
+		if err := removeDeploymentFinalizers(log, d.deps.DatabaseCRCli, updated, removalList); err != nil {
+			log.Debug().Err(err).Msg("Failed to update ArangoDeployment (to remove finalizers)")
+			return maskAny(err)
+		}
+	}
+	return nil
+}
+
+// inspectRemoveChildFinalizers checks the finalizer condition for remove-child-finalizers.
+// It returns nil if the finalizer can be removed.
+func (d *Deployment) inspectRemoveChildFinalizers(ctx context.Context, log zerolog.Logger, depl *api.ArangoDeployment) error {
+	if err := d.removePodFinalizers(); err != nil {
+		return maskAny(err)
+	}
+	if err := d.removePVCFinalizers(); err != nil {
+		return maskAny(err)
+	}
+
+	return nil
+}
+
+// removeDeploymentFinalizers removes the given finalizers from the given PVC.
+func removeDeploymentFinalizers(log zerolog.Logger, cli versioned.Interface, depl *api.ArangoDeployment, finalizers []string) error {
+	depls := cli.DatabaseV1alpha().ArangoDeployments(depl.GetNamespace())
+	getFunc := func() (metav1.Object, error) {
+		result, err := depls.Get(depl.GetName(), metav1.GetOptions{})
+		if err != nil {
+			return nil, maskAny(err)
+		}
+		return result, nil
+	}
+	updateFunc := func(updated metav1.Object) error {
+		updatedDepl := updated.(*api.ArangoDeployment)
+		result, err := depls.Update(updatedDepl)
+		if err != nil {
+			return maskAny(err)
+		}
+		*depl = *result
+		return nil
+	}
+	if err := k8sutil.RemoveFinalizers(log, finalizers, getFunc, updateFunc); err != nil {
+		return maskAny(err)
+	}
+	return nil
+}

pkg/deployment/deployment_inspector.go

@@ -46,13 +46,19 @@ func (d *Deployment) inspectDeployment(lastInterval time.Duration) time.Duration
 	ctx := context.Background()
 
 	// Check deployment still exists
-	if _, err := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(d.apiObject.GetNamespace()).Get(d.apiObject.GetName(), metav1.GetOptions{}); k8sutil.IsNotFound(err) {
+	updated, err := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(d.apiObject.GetNamespace()).Get(d.apiObject.GetName(), metav1.GetOptions{})
+	if k8sutil.IsNotFound(err) {
 		// Deployment is gone
 		log.Info().Msg("Deployment is gone")
 		d.Delete()
 		return nextInterval
+	} else if updated != nil && updated.GetDeletionTimestamp() != nil {
+		// Deployment is marked for deletion
+		if err := d.runDeploymentFinalizers(ctx); err != nil {
+			hasError = true
+			d.CreateEvent(k8sutil.NewErrorEvent("ArangoDeployment finalizer inspection failed", err, d.apiObject))
 		}
-
+	} else {
 		// Is the deployment in failed state, if so, give up.
 		if d.status.Phase == api.DeploymentPhaseFailed {
 			log.Debug().Msg("Deployment is in Failed state.")
@@ -146,6 +152,7 @@ func (d *Deployment) inspectDeployment(lastInterval time.Duration) time.Duration
 			hasError = true
 			d.CreateEvent(k8sutil.NewErrorEvent("Pod cleanup failed", err, d.apiObject))
 		}
+	}
 
 	// Update next interval (on errors)
 	if hasError {

pkg/util/constants/constants.go

@@ -44,10 +44,11 @@ const (
 
 	SecretAccessPackageYaml = "accessPackage.yaml" // Key in Secret.data used to store a YAML encoded access package
 
-	FinalizerPodDrainDBServer = "dbserver.database.arangodb.com/drain"        // Finalizer added to DBServers, indicating the need for draining that dbserver
-	FinalizerPodAgencyServing = "agent.database.arangodb.com/agency-serving"  // Finalizer added to Agents, indicating the need for keeping enough agents alive
-	FinalizerPVCMemberExists  = "pvc.database.arangodb.com/member-exists"     // Finalizer added to PVCs, indicating the need to keep is as long as its member exists
+	FinalizerDeplRemoveChildFinalizers = "database.arangodb.com/remove-child-finalizers" // Finalizer added to ArangoDeployment, indicating the need to remove finalizers from all children
 	FinalizerDeplReplStopSync          = "replication.database.arangodb.com/stop-sync"   // Finalizer added to ArangoDeploymentReplication, indicating the need to stop synchronization
+	FinalizerPodAgencyServing          = "agent.database.arangodb.com/agency-serving"    // Finalizer added to Agents, indicating the need for keeping enough agents alive
+	FinalizerPodDrainDBServer          = "dbserver.database.arangodb.com/drain"          // Finalizer added to DBServers, indicating the need for draining that dbserver
+	FinalizerPVCMemberExists           = "pvc.database.arangodb.com/member-exists"       // Finalizer added to PVCs, indicating the need to keep is as long as its member exists
 
 	AnnotationEnforceAntiAffinity = "database.arangodb.com/enforce-anti-affinity" // Key of annotation added to PVC. Value is a boolean "true" or "false"
 )