diff --git a/CHANGELOG.md b/CHANGELOG.md index b6917d854..9680541c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - (Bugfix) Do not block reconciliation in case of Resource failure - (Improvement) Multi-arch support for ID member - (Feature) Allow to change Pod Network and PID settings +- (Feature) Pre OOM Abort function ## [1.2.20](https://github.com/arangodb/kube-arangodb/tree/1.2.20) (2022-10-25) - (Feature) Add action progress diff --git a/cmd/cmd.go b/cmd/cmd.go index 091ad1cc0..7ccb96606 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -97,6 +97,10 @@ var ( Run: executeMain, } + memoryLimit struct { + hardLimit uint64 + } + logLevels []string serverOptions struct { host string @@ -209,6 +213,7 @@ func init() { f.IntVar(&operatorKubernetesOptions.burst, "kubernetes.burst", kclient.DefaultBurst, "Burst for the k8s API") f.BoolVar(&crdOptions.install, "crd.install", true, "Install missing CRD if access is possible") f.IntVar(&operatorBackup.concurrentUploads, "backup-concurrent-uploads", globals.DefaultBackupConcurrentUploads, "Number of concurrent uploads per deployment") + f.Uint64Var(&memoryLimit.hardLimit, "memory-limit", 0, "Define memory limit for hard shutdown and the dump of goroutines. Used for testing") if err := features.Init(&cmdMain); err != nil { panic(err.Error()) } @@ -236,6 +241,8 @@ func executeMain(cmd *cobra.Command, args []string) { name := os.Getenv(constants.EnvOperatorPodName) ip := os.Getenv(constants.EnvOperatorPodIP) + go monitorMemoryLimit() + deploymentApi.DefaultImage = operatorOptions.arangoImage globals.GetGlobalTimeouts().Kubernetes().Set(operatorTimeouts.k8s) diff --git a/cmd/memory.go b/cmd/memory.go new file mode 100644 index 000000000..183856f80 --- /dev/null +++ b/cmd/memory.go @@ -0,0 +1,55 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package cmd + +import ( + "runtime" + "syscall" + "time" + + "github.com/arangodb/kube-arangodb/pkg/util/shutdown" +) + +func monitorMemoryLimit() { + if memoryLimit.hardLimit == 0 { + return + } + + var m runtime.MemStats + + t := time.NewTicker(time.Millisecond) + defer t.Stop() + + for { + select { + case <-t.C: + runtime.ReadMemStats(&m) + + if m.Sys > 1024*1024*memoryLimit.hardLimit { + if err := syscall.Kill(syscall.Getpid(), syscall.SIGABRT); err != nil { + panic(err) + } + } + case <-shutdown.Channel(): + return + } + } +} diff --git a/pkg/util/shutdown/shutdown.go b/pkg/util/shutdown/shutdown.go new file mode 100644 index 000000000..6ed28944c --- /dev/null +++ b/pkg/util/shutdown/shutdown.go @@ -0,0 +1,46 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package shutdown + +import ( + "os" + "os/signal" + "syscall" +) + +func init() { + shutdown = make(chan struct{}) + + sigChannel := make(chan os.Signal, 2) + + signal.Notify(sigChannel, os.Interrupt, syscall.SIGTERM) + + go func() { + defer close(shutdown) + <-sigChannel + }() +} + +var shutdown chan struct{} + +func Channel() <-chan struct{} { + return shutdown +} diff --git a/pkg/util/signal.go b/pkg/util/signal.go index 998235297..5b6784f02 100644 --- a/pkg/util/signal.go +++ b/pkg/util/signal.go @@ -22,30 +22,21 @@ package util import ( "context" - "os" - "os/signal" - "syscall" + + "github.com/arangodb/kube-arangodb/pkg/util/shutdown" ) // CreateSignalContext creates and returns the context which is closed when one of the provided signal occurs. -// If the provided list of signals is empty, then SIGINT and SIGTERM is used by default. -func CreateSignalContext(ctx context.Context, signals ...os.Signal) context.Context { +// SIGINT and SIGTERM is used by default. +func CreateSignalContext(ctx context.Context) context.Context { if ctx == nil { ctx = context.Background() } ctxSignal, cancelSignal := context.WithCancel(ctx) - sigChannel := make(chan os.Signal, 2) - - if len(signals) > 0 { - signal.Notify(sigChannel, signals...) - } else { - signal.Notify(sigChannel, os.Interrupt, syscall.SIGTERM) - } go func() { - // Wait until signal occurs. - <-sigChannel - close(sigChannel) + // Wait until shutdown starts + <-shutdown.Channel() // Close the context which is used by the caller. cancelSignal() }()