2018-05-11 08:07:00 +00:00
|
|
|
//
|
|
|
|
// DISCLAIMER
|
|
|
|
//
|
2022-01-10 11:35:49 +00:00
|
|
|
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
|
2018-05-11 08:07:00 +00:00
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
//
|
|
|
|
// Copyright holder is ArangoDB GmbH, Cologne, Germany
|
|
|
|
//
|
|
|
|
|
|
|
|
package resources
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2018-05-15 06:42:05 +00:00
|
|
|
"time"
|
2018-05-11 08:07:00 +00:00
|
|
|
|
2021-12-10 12:04:48 +00:00
|
|
|
"github.com/arangodb/kube-arangodb/pkg/util/globals"
|
|
|
|
|
2021-01-08 14:35:38 +00:00
|
|
|
"github.com/arangodb/kube-arangodb/pkg/util/errors"
|
|
|
|
|
2020-03-10 09:26:38 +00:00
|
|
|
v1 "k8s.io/api/core/v1"
|
2022-05-03 08:31:30 +00:00
|
|
|
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
|
2018-05-11 08:07:00 +00:00
|
|
|
|
2019-11-04 07:49:24 +00:00
|
|
|
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
|
2018-08-30 14:57:08 +00:00
|
|
|
"github.com/arangodb/kube-arangodb/pkg/util"
|
2018-05-11 08:07:00 +00:00
|
|
|
"github.com/arangodb/kube-arangodb/pkg/util/constants"
|
|
|
|
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
|
|
|
|
)
|
|
|
|
|
2018-08-25 10:08:44 +00:00
|
|
|
const (
|
2018-08-30 14:57:08 +00:00
|
|
|
podFinalizerRemovedInterval = util.Interval(time.Second / 2) // Interval used (until new inspection) when Pod finalizers have been removed
|
|
|
|
recheckPodFinalizerInterval = util.Interval(time.Second * 10) // Interval used when Pod finalizers need to be rechecked soon
|
2022-02-09 12:32:06 +00:00
|
|
|
podUnreachableGracePeriod = time.Second * 15 // Interval used when Pod finalizers need to be rechecked soon
|
2018-08-25 10:08:44 +00:00
|
|
|
)
|
|
|
|
|
2018-05-11 08:07:00 +00:00
|
|
|
// runPodFinalizers goes through the list of pod finalizers to see if they can be removed.
|
2018-08-25 10:08:44 +00:00
|
|
|
// Returns: Interval_till_next_inspection, error
|
2018-08-30 14:57:08 +00:00
|
|
|
func (r *Resources) runPodFinalizers(ctx context.Context, p *v1.Pod, memberStatus api.MemberStatus, updateMember func(api.MemberStatus) error) (util.Interval, error) {
|
2022-06-14 07:26:07 +00:00
|
|
|
log := r.log.Str("section", "pod").Str("pod-name", p.GetName())
|
2018-05-11 08:07:00 +00:00
|
|
|
var removalList []string
|
2021-07-04 16:16:22 +00:00
|
|
|
|
2021-12-29 21:04:08 +00:00
|
|
|
// When the main container is terminated, then the whole pod should be terminated,
|
|
|
|
// so sidecar core containers' names should not be checked here.
|
2022-02-03 23:03:12 +00:00
|
|
|
// If Member is not reachable finalizers should be also removed
|
2022-02-09 12:32:06 +00:00
|
|
|
isServerContainerDead := !k8sutil.IsPodServerContainerRunning(p) || memberStatus.Conditions.Check(api.ConditionTypeReachable).Exists().IsFalse().LastTransition(podUnreachableGracePeriod).Evaluate()
|
2021-07-04 16:16:22 +00:00
|
|
|
|
2018-05-11 08:07:00 +00:00
|
|
|
for _, f := range p.ObjectMeta.GetFinalizers() {
|
|
|
|
switch f {
|
2018-05-14 08:50:06 +00:00
|
|
|
case constants.FinalizerPodAgencyServing:
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Debug("Inspecting agency-serving finalizer")
|
2021-07-04 16:16:22 +00:00
|
|
|
if isServerContainerDead {
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Debug("Server Container is dead, removing finalizer")
|
2021-07-04 16:16:22 +00:00
|
|
|
removalList = append(removalList, f)
|
|
|
|
break
|
|
|
|
}
|
2022-06-14 07:26:07 +00:00
|
|
|
if err := r.inspectFinalizerPodAgencyServing(ctx, p, memberStatus, updateMember); err == nil {
|
2018-05-14 08:50:06 +00:00
|
|
|
removalList = append(removalList, f)
|
|
|
|
} else {
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Err(err).Str("finalizer", f).Debug("Cannot remove finalizer yet")
|
2018-05-14 08:50:06 +00:00
|
|
|
}
|
2018-05-11 14:56:20 +00:00
|
|
|
case constants.FinalizerPodDrainDBServer:
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Debug("Inspecting drain dbserver finalizer")
|
2021-07-04 16:16:22 +00:00
|
|
|
if isServerContainerDead {
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Debug("Server Container is dead, removing finalizer")
|
2021-07-04 16:16:22 +00:00
|
|
|
removalList = append(removalList, f)
|
|
|
|
break
|
|
|
|
}
|
2022-06-14 07:26:07 +00:00
|
|
|
if err := r.inspectFinalizerPodDrainDBServer(ctx, p, memberStatus, updateMember); err == nil {
|
2018-05-11 08:07:00 +00:00
|
|
|
removalList = append(removalList, f)
|
|
|
|
} else {
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Err(err).Str("finalizer", f).Debug("Cannot remove Pod finalizer yet")
|
2018-05-11 08:07:00 +00:00
|
|
|
}
|
2021-10-29 07:41:41 +00:00
|
|
|
case constants.FinalizerPodGracefulShutdown:
|
|
|
|
// We are in graceful shutdown, only one way to remove it is when container is already dead
|
|
|
|
if isServerContainerDead {
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Debug("Server Container is dead, removing finalizer")
|
2021-10-29 07:41:41 +00:00
|
|
|
removalList = append(removalList, f)
|
|
|
|
}
|
2021-04-06 12:42:34 +00:00
|
|
|
case constants.FinalizerDelayPodTermination:
|
2021-07-04 16:16:22 +00:00
|
|
|
if isServerContainerDead {
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Debug("Server Container is dead, removing finalizer")
|
2021-07-04 16:16:22 +00:00
|
|
|
removalList = append(removalList, f)
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
2021-04-06 12:42:34 +00:00
|
|
|
s, _ := r.context.GetStatus()
|
|
|
|
_, group, ok := s.Members.ElementByID(memberStatus.ID)
|
|
|
|
if !ok {
|
|
|
|
continue
|
|
|
|
}
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Str("finalizer", f).Error("Delay finalizer")
|
2021-04-06 12:42:34 +00:00
|
|
|
|
|
|
|
groupSpec := r.context.GetSpec().GetServerGroupSpec(group)
|
|
|
|
if t := p.ObjectMeta.DeletionTimestamp; t != nil {
|
2021-10-29 07:41:41 +00:00
|
|
|
d := time.Duration(groupSpec.GetShutdownDelay(group)) * time.Second
|
|
|
|
gr := time.Duration(util.Int64OrDefault(p.ObjectMeta.GetDeletionGracePeriodSeconds(), 0)) * time.Second
|
|
|
|
e := t.Time.Add(-1 * gr).Sub(time.Now().Add(-1 * d))
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Str("finalizer", f).Str("left", e.String()).Error("Delay finalizer status")
|
2021-10-29 07:41:41 +00:00
|
|
|
if e < 0 || d == 0 {
|
2021-04-06 12:42:34 +00:00
|
|
|
removalList = append(removalList, f)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
continue
|
|
|
|
}
|
2018-05-11 08:07:00 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
// Remove finalizers (if needed)
|
|
|
|
if len(removalList) > 0 {
|
2022-06-14 07:26:07 +00:00
|
|
|
if _, err := k8sutil.RemovePodFinalizers(ctx, r.context.ACS().CurrentClusterCache(), r.context.ACS().CurrentClusterCache().PodsModInterface().V1(), p, removalList, false); err != nil {
|
|
|
|
log.Err(err).Debug("Failed to update pod (to remove finalizers)")
|
2021-01-08 14:35:38 +00:00
|
|
|
return 0, errors.WithStack(err)
|
2018-05-11 08:07:00 +00:00
|
|
|
}
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Strs("finalizers", removalList...).Debug("Removed finalizer(s) from Pod")
|
2018-08-30 14:57:08 +00:00
|
|
|
// Let's do the next inspection quickly, since things may have changed now.
|
|
|
|
return podFinalizerRemovedInterval, nil
|
2018-05-11 08:07:00 +00:00
|
|
|
}
|
2018-08-30 14:57:08 +00:00
|
|
|
// Check again at given interval
|
|
|
|
return recheckPodFinalizerInterval, nil
|
2018-05-11 08:07:00 +00:00
|
|
|
}
|
|
|
|
|
2018-05-14 08:50:06 +00:00
|
|
|
// inspectFinalizerPodAgencyServing checks the finalizer condition for agency-serving.
|
|
|
|
// It returns nil if the finalizer can be removed.
|
2022-06-14 07:26:07 +00:00
|
|
|
func (r *Resources) inspectFinalizerPodAgencyServing(ctx context.Context, p *v1.Pod, memberStatus api.MemberStatus, updateMember func(api.MemberStatus) error) error {
|
|
|
|
log := r.log.Str("section", "agency")
|
2022-06-20 14:43:23 +00:00
|
|
|
if err := r.prepareAgencyPodTermination(p, memberStatus, func(update api.MemberStatus) error {
|
2018-08-30 08:15:42 +00:00
|
|
|
if err := updateMember(update); err != nil {
|
2021-01-08 14:35:38 +00:00
|
|
|
return errors.WithStack(err)
|
2018-08-30 08:15:42 +00:00
|
|
|
}
|
|
|
|
memberStatus = update
|
|
|
|
return nil
|
|
|
|
}); err != nil {
|
2018-08-27 07:52:43 +00:00
|
|
|
// Pod cannot be terminated yet
|
2021-01-08 14:35:38 +00:00
|
|
|
return errors.WithStack(err)
|
2018-05-14 08:50:06 +00:00
|
|
|
}
|
2018-05-14 15:07:12 +00:00
|
|
|
|
2018-08-30 08:15:42 +00:00
|
|
|
// Remaining agents are healthy, if we need to perform complete recovery
|
|
|
|
// of the agent, also remove the PVC
|
|
|
|
if memberStatus.Conditions.IsTrue(api.ConditionTypeAgentRecoveryNeeded) {
|
2021-12-10 12:04:48 +00:00
|
|
|
err := globals.GetGlobalTimeouts().Kubernetes().RunWithTimeout(ctx, func(ctxChild context.Context) error {
|
2022-05-31 11:31:00 +00:00
|
|
|
return r.context.ACS().CurrentClusterCache().PersistentVolumeClaimsModInterface().V1().Delete(ctxChild, memberStatus.PersistentVolumeClaimName, meta.DeleteOptions{})
|
2021-04-26 08:30:06 +00:00
|
|
|
})
|
|
|
|
if err != nil && !k8sutil.IsNotFound(err) {
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Err(err).Warn("Failed to delete PVC for member")
|
2021-01-08 14:35:38 +00:00
|
|
|
return errors.WithStack(err)
|
2018-08-30 08:15:42 +00:00
|
|
|
}
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Str("pvc-name", memberStatus.PersistentVolumeClaimName).Debug("Removed PVC of member so agency can be completely replaced")
|
2018-05-14 15:07:12 +00:00
|
|
|
}
|
|
|
|
|
2018-05-14 08:50:06 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-05-11 14:56:20 +00:00
|
|
|
// inspectFinalizerPodDrainDBServer checks the finalizer condition for drain-dbserver.
|
2018-05-11 08:07:00 +00:00
|
|
|
// It returns nil if the finalizer can be removed.
|
2022-06-14 07:26:07 +00:00
|
|
|
func (r *Resources) inspectFinalizerPodDrainDBServer(ctx context.Context, p *v1.Pod, memberStatus api.MemberStatus, updateMember func(api.MemberStatus) error) error {
|
|
|
|
log := r.log.Str("section", "pod")
|
|
|
|
if err := r.prepareDBServerPodTermination(ctx, p, memberStatus, func(update api.MemberStatus) error {
|
2018-08-30 08:15:42 +00:00
|
|
|
if err := updateMember(update); err != nil {
|
2021-01-08 14:35:38 +00:00
|
|
|
return errors.WithStack(err)
|
2018-08-30 08:15:42 +00:00
|
|
|
}
|
|
|
|
memberStatus = update
|
|
|
|
return nil
|
|
|
|
}); err != nil {
|
2018-08-27 08:24:19 +00:00
|
|
|
// Pod cannot be terminated yet
|
2021-01-08 14:35:38 +00:00
|
|
|
return errors.WithStack(err)
|
2018-05-11 14:56:20 +00:00
|
|
|
}
|
2018-05-14 15:07:12 +00:00
|
|
|
|
2018-08-30 08:15:42 +00:00
|
|
|
// If this DBServer is cleaned out, we need to remove the PVC.
|
2019-02-08 15:09:46 +00:00
|
|
|
if memberStatus.Conditions.IsTrue(api.ConditionTypeCleanedOut) || memberStatus.Phase == api.MemberPhaseDrain {
|
2021-12-10 12:04:48 +00:00
|
|
|
err := globals.GetGlobalTimeouts().Kubernetes().RunWithTimeout(ctx, func(ctxChild context.Context) error {
|
2022-05-31 11:31:00 +00:00
|
|
|
return r.context.ACS().CurrentClusterCache().PersistentVolumeClaimsModInterface().V1().Delete(ctxChild, memberStatus.PersistentVolumeClaimName, meta.DeleteOptions{})
|
2021-04-26 08:30:06 +00:00
|
|
|
})
|
|
|
|
if err != nil && !k8sutil.IsNotFound(err) {
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Err(err).Warn("Failed to delete PVC for member")
|
2021-01-08 14:35:38 +00:00
|
|
|
return errors.WithStack(err)
|
2018-08-30 08:15:42 +00:00
|
|
|
}
|
2022-06-14 07:26:07 +00:00
|
|
|
log.Str("pvc-name", memberStatus.PersistentVolumeClaimName).Debug("Removed PVC of member")
|
2018-05-11 08:07:00 +00:00
|
|
|
}
|
2018-05-14 15:07:12 +00:00
|
|
|
|
2018-08-27 08:24:19 +00:00
|
|
|
return nil
|
2018-05-11 08:07:00 +00:00
|
|
|
}
|