mirrors/prometheus-operator
mirrors/prometheus-operator
1
0
Fork 0
mirror of https://github.com/prometheus-operator/prometheus-operator.git synced 2025-04-21 11:48:53 +00:00
Code Activity

pkg/alertmanager: change podManagement policy to parallel to prevent statefulset reconciliation from hanging

Browse source 
When using default podManagementPolicy it is possible to create a
situation where alertmanager pods objects won't be reconciled with a
statefulset and thus preventing am from being deployed.

One of such cases is when am was deployed and afterwards admin
applied taints to all nodes causing pod eviction. Next tolerations were
applied however due to OrderedReady policy one am pod was still left in
Pending state preventing reconciliation.

This is needed to provide a workaround for a bug in kubernetes detailed
in https://github.com/kubernetes/kubernetes/issues/60164. It is also one
of the knows limitations of StetafulSets mentioned in docs
https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#limitations
This commit is contained in:
paulfantom 2019-07-17 12:13:13 +02:00
parent 7c448d90c1
commit ac7626c151
No known key found for this signature in database
GPG key ID: 12AE0185401674E7
2 changed files with 8 additions and 4 deletions
pkg
alertmanager
statefulset.go
prometheus
statefulset.go

9
pkg/alertmanager/statefulset.go
View file

@ -21,7 +21,7 @@ import (
"strings"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
@ -444,9 +444,12 @@ func makeStatefulSetSpec(a *monitoringv1.Alertmanager, config Config) (*appsv1.S
terminationGracePeriod := int64(120)
finalLabels := config.Labels.Merge(podLabels)
// PodManagementPolicy is set to Parallel to mitigate issues in kuberentes: https://github.com/kubernetes/kubernetes/issues/60164
// This is also mentioned as one of limitations of StatefulSets: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#limitations
return &appsv1.StatefulSetSpec{
ServiceName: governingServiceName,
Replicas: a.Spec.Replicas,
ServiceName: governingServiceName,
Replicas: a.Spec.Replicas,
PodManagementPolicy: appsv1.ParallelPodManagement,
UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
Type: appsv1.RollingUpdateStatefulSetStrategyType,
},

3
pkg/prometheus/statefulset.go
View file

@ -798,7 +798,8 @@ func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config, ruleConfigMapName
if err != nil {
return nil, errors.Wrap(err, "failed to merge containers spec")
}
// PodManagementPolicy is set to Parallel to mitigate issues in kuberentes: https://github.com/kubernetes/kubernetes/issues/60164
// This is also mentioned as one of limitations of StatefulSets: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#limitations
return &appsv1.StatefulSetSpec{
ServiceName: governingServiceName,
Replicas: p.Spec.Replicas,