2020-06-08 17:34:52 +00:00
groups :
- name : kubernetes-apps
rules :
- alert : KubePodCrashLooping
annotations :
2021-10-09 03:22:59 +00:00
description : 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
}}) is in waiting state (reason : "CrashLoopBackOff" ).'
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping
2020-08-17 03:17:41 +00:00
summary : Pod is crash looping.
2020-06-08 17:34:52 +00:00
expr : |
2021-10-09 03:22:59 +00:00
max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="kube-state-metrics"}[5m]) >= 1
2020-06-08 17:34:52 +00:00
for : 15m
labels :
severity : warning
- alert : KubePodNotReady
annotations :
2020-08-17 03:17:41 +00:00
description : Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready
2020-08-13 10:50:10 +00:00
state for longer than 15 minutes.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready
2020-08-17 03:17:41 +00:00
summary : Pod has been in a non-ready state for more than 15 minutes.
2020-06-08 17:34:52 +00:00
expr : |
sum by (namespace, pod) (
max by(namespace, pod) (
kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown"}
) * on(namespace, pod) group_left(owner_kind) topk by(namespace, pod) (
1 , max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"})
)
) > 0
for : 15m
labels :
severity : warning
- alert : KubeDeploymentGenerationMismatch
annotations :
2020-08-17 03:17:41 +00:00
description : Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment
2020-08-13 10:50:10 +00:00
}} does not match, this indicates that the Deployment has failed but has not
been rolled back.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch
2020-08-17 03:17:41 +00:00
summary : Deployment generation mismatch due to possible roll-back
2020-06-08 17:34:52 +00:00
expr : |
kube_deployment_status_observed_generation{job="kube-state-metrics"}
!=
kube_deployment_metadata_generation{job="kube-state-metrics"}
for : 15m
labels :
severity : warning
- alert : KubeDeploymentReplicasMismatch
annotations :
2020-08-17 03:17:41 +00:00
description : Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has
not matched the expected number of replicas for longer than 15 minutes.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch
2020-08-17 03:17:41 +00:00
summary : Deployment has not matched the expected number of replicas.
2020-06-08 17:34:52 +00:00
expr : |
(
kube_deployment_spec_replicas{job="kube-state-metrics"}
2021-07-01 03:25:31 +00:00
>
2020-06-08 17:34:52 +00:00
kube_deployment_status_replicas_available{job="kube-state-metrics"}
) and (
2021-02-03 03:27:36 +00:00
changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m])
2020-06-08 17:34:52 +00:00
==
0
)
for : 15m
labels :
severity : warning
- alert : KubeStatefulSetReplicasMismatch
annotations :
2020-08-17 03:17:41 +00:00
description : StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has
not matched the expected number of replicas for longer than 15 minutes.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch
2020-08-17 03:17:41 +00:00
summary : Deployment has not matched the expected number of replicas.
2020-06-08 17:34:52 +00:00
expr : |
(
kube_statefulset_status_replicas_ready{job="kube-state-metrics"}
!=
kube_statefulset_status_replicas{job="kube-state-metrics"}
) and (
2021-02-03 03:27:36 +00:00
changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[10m])
2020-06-08 17:34:52 +00:00
==
0
)
for : 15m
labels :
severity : warning
- alert : KubeStatefulSetGenerationMismatch
annotations :
2020-08-17 03:17:41 +00:00
description : StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset
2020-08-13 10:50:10 +00:00
}} does not match, this indicates that the StatefulSet has failed but has
not been rolled back.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch
2020-08-17 03:17:41 +00:00
summary : StatefulSet generation mismatch due to possible roll-back
2020-06-08 17:34:52 +00:00
expr : |
kube_statefulset_status_observed_generation{job="kube-state-metrics"}
!=
kube_statefulset_metadata_generation{job="kube-state-metrics"}
for : 15m
labels :
severity : warning
- alert : KubeStatefulSetUpdateNotRolledOut
annotations :
2020-08-17 03:17:41 +00:00
description : StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update
2020-08-13 10:50:10 +00:00
has not been rolled out.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout
2020-08-17 03:17:41 +00:00
summary : StatefulSet update has not been rolled out.
2020-06-08 17:34:52 +00:00
expr : |
(
2020-06-24 10:14:37 +00:00
max without (revision) (
kube_statefulset_status_current_revision{job="kube-state-metrics"}
unless
kube_statefulset_status_update_revision{job="kube-state-metrics"}
)
*
(
kube_statefulset_replicas{job="kube-state-metrics"}
!=
kube_statefulset_status_replicas_updated{job="kube-state-metrics"}
)
) and (
changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m])
==
0
2020-06-08 17:34:52 +00:00
)
for : 15m
labels :
severity : warning
- alert : KubeDaemonSetRolloutStuck
annotations :
2020-08-17 03:17:41 +00:00
description : DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not
finished or progressed for at least 15 minutes.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck
2020-08-17 03:17:41 +00:00
summary : DaemonSet rollout is stuck.
2020-06-08 17:34:52 +00:00
expr : |
2020-08-12 17:33:36 +00:00
(
(
kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"}
!=
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
) or (
kube_daemonset_status_number_misscheduled{job="kube-state-metrics"}
!=
0
) or (
kube_daemonset_updated_number_scheduled{job="kube-state-metrics"}
!=
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
) or (
kube_daemonset_status_number_available{job="kube-state-metrics"}
!=
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
)
) and (
changes(kube_daemonset_updated_number_scheduled{job="kube-state-metrics"}[5m])
==
0
)
2020-06-08 17:34:52 +00:00
for : 15m
labels :
severity : warning
- alert : KubeContainerWaiting
annotations :
2020-08-17 03:17:41 +00:00
description : Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}}
2020-08-13 10:50:10 +00:00
has been in waiting state for longer than 1 hour.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting
2020-08-17 03:17:41 +00:00
summary : Pod container waiting longer than 1 hour
2020-06-08 17:34:52 +00:00
expr : |
sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0
for : 1h
labels :
severity : warning
- alert : KubeDaemonSetNotScheduled
annotations :
2020-08-17 03:17:41 +00:00
description : '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset
2020-08-13 10:50:10 +00:00
}} are not scheduled.'
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled
2020-08-17 03:17:41 +00:00
summary : DaemonSet pods are not scheduled.
2020-06-08 17:34:52 +00:00
expr : |
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
-
kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0
for : 10m
labels :
severity : warning
- alert : KubeDaemonSetMisScheduled
annotations :
2020-08-17 03:17:41 +00:00
description : '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset
2020-08-13 10:50:10 +00:00
}} are running where they are not supposed to run.'
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled
2020-08-17 03:17:41 +00:00
summary : DaemonSet pods are misscheduled.
2020-06-08 17:34:52 +00:00
expr : |
kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0
for : 15m
labels :
severity : warning
- alert : KubeJobCompletion
annotations :
2020-08-17 03:17:41 +00:00
description : Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more
than 12 hours to complete.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion
2020-08-18 13:43:59 +00:00
summary : Job did not complete in time
2020-06-08 17:34:52 +00:00
expr : |
kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"} > 0
2020-08-12 17:33:36 +00:00
for : 12h
2020-06-08 17:34:52 +00:00
labels :
severity : warning
- alert : KubeJobFailed
annotations :
2020-08-17 03:17:41 +00:00
description : Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete.
2020-10-10 03:23:27 +00:00
Removing failed job after investigation should clear this alert.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed
2020-08-17 03:17:41 +00:00
summary : Job failed to complete.
2020-06-08 17:34:52 +00:00
expr : |
kube_job_failed{job="kube-state-metrics"} > 0
for : 15m
labels :
severity : warning
- alert : KubeHpaReplicasMismatch
annotations :
2021-07-07 03:24:56 +00:00
description : HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }}
has not matched the desired number of replicas for longer than 15 minutes.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch
2020-08-17 03:17:41 +00:00
summary : HPA has not matched descired number of replicas.
2020-06-08 17:34:52 +00:00
expr : |
2021-06-15 03:30:31 +00:00
(kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics"}
2020-06-08 17:34:52 +00:00
!=
2021-06-15 03:30:31 +00:00
kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"})
2020-11-01 03:18:27 +00:00
and
2021-06-15 03:30:31 +00:00
(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
2020-11-01 03:18:27 +00:00
>
2021-06-15 03:30:31 +00:00
kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics"})
2020-11-01 03:18:27 +00:00
and
2021-06-15 03:30:31 +00:00
(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
2020-11-01 03:18:27 +00:00
<
2021-06-15 03:30:31 +00:00
kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"})
2020-06-08 17:34:52 +00:00
and
2021-06-15 03:30:31 +00:00
changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}[15m]) == 0
2020-06-08 17:34:52 +00:00
for : 15m
labels :
severity : warning
- alert : KubeHpaMaxedOut
annotations :
2021-07-07 03:24:56 +00:00
description : HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }}
has been running at max replicas for longer than 15 minutes.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout
2020-08-17 03:17:41 +00:00
summary : HPA is running at max replicas
2020-06-08 17:34:52 +00:00
expr : |
2021-06-15 03:30:31 +00:00
kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
2020-06-08 17:34:52 +00:00
==
2021-06-15 03:30:31 +00:00
kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"}
2020-06-08 17:34:52 +00:00
for : 15m
labels :
severity : warning
- name : kubernetes-resources
rules :
- alert : KubeCPUOvercommit
annotations :
2021-08-03 03:25:13 +00:00
description : Cluster has overcommitted CPU resource requests for Pods by {{
$value }} CPU shares and cannot tolerate node failure.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit
2020-08-17 03:17:41 +00:00
summary : Cluster has overcommitted CPU resource requests.
2020-06-08 17:34:52 +00:00
expr : |
2021-08-03 03:25:13 +00:00
sum(namespace_cpu:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="cpu"}) - max(kube_node_status_allocatable{resource="cpu"})) > 0
and
(sum(kube_node_status_allocatable{resource="cpu"}) - max(kube_node_status_allocatable{resource="cpu"})) > 0
for : 10m
2020-06-08 17:34:52 +00:00
labels :
severity : warning
- alert : KubeMemoryOvercommit
annotations :
2021-08-03 03:25:13 +00:00
description : Cluster has overcommitted memory resource requests for Pods by
{{ $value }} bytes and cannot tolerate node failure.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryovercommit
2020-08-17 03:17:41 +00:00
summary : Cluster has overcommitted memory resource requests.
2020-06-08 17:34:52 +00:00
expr : |
2021-08-03 03:25:13 +00:00
sum(namespace_memory:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0
and
(sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0
for : 10m
2020-06-08 17:34:52 +00:00
labels :
severity : warning
- alert : KubeCPUQuotaOvercommit
annotations :
2020-08-17 03:17:41 +00:00
description : Cluster has overcommitted CPU resource requests for Namespaces.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuquotaovercommit
2020-08-17 03:17:41 +00:00
summary : Cluster has overcommitted CPU resource requests.
2021-04-14 03:32:10 +00:00
expr : |
sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"})
/
sum(kube_node_status_allocatable{resource="cpu"})
> 1.5
2020-06-08 17:34:52 +00:00
for : 5m
labels :
severity : warning
- alert : KubeMemoryQuotaOvercommit
annotations :
2020-08-17 03:17:41 +00:00
description : Cluster has overcommitted memory resource requests for Namespaces.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryquotaovercommit
2020-08-17 03:17:41 +00:00
summary : Cluster has overcommitted memory resource requests.
2020-06-08 17:34:52 +00:00
expr : |
sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"})
/
2021-03-26 04:18:04 +00:00
sum(kube_node_status_allocatable{resource="memory",job="kube-state-metrics"})
2020-06-08 17:34:52 +00:00
> 1.5
for : 5m
labels :
severity : warning
2020-09-11 03:19:54 +00:00
- alert : KubeQuotaAlmostFull
annotations :
description : Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
}} of its {{ $labels.resource }} quota.
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaalmostfull
summary : Namespace quota is going to be full.
expr : |
kube_resourcequota{job="kube-state-metrics", type="used"}
/ ignoring(instance, job, type)
(kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
> 0.9 < 1
for : 15m
labels :
severity : info
2020-07-01 09:26:49 +00:00
- alert : KubeQuotaFullyUsed
2020-06-08 17:34:52 +00:00
annotations :
2020-08-17 03:17:41 +00:00
description : Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
2020-08-13 10:50:10 +00:00
}} of its {{ $labels.resource }} quota.
2020-07-01 09:26:49 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotafullyused
2020-08-17 03:17:41 +00:00
summary : Namespace quota is fully used.
2020-06-08 17:34:52 +00:00
expr : |
kube_resourcequota{job="kube-state-metrics", type="used"}
/ ignoring(instance, job, type)
(kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
2020-09-11 03:19:54 +00:00
== 1
2020-06-08 17:34:52 +00:00
for : 15m
labels :
2020-07-01 09:26:49 +00:00
severity : info
2020-09-11 03:19:54 +00:00
- alert : KubeQuotaExceeded
annotations :
description : Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
}} of its {{ $labels.resource }} quota.
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded
summary : Namespace quota has exceeded the limits.
expr : |
kube_resourcequota{job="kube-state-metrics", type="used"}
/ ignoring(instance, job, type)
(kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
> 1
for : 15m
labels :
severity : warning
2020-06-08 17:34:52 +00:00
- alert : CPUThrottlingHigh
annotations :
2020-08-17 03:17:41 +00:00
description : '{{ $value | humanizePercentage }} throttling of CPU in namespace
{{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod
2020-08-13 10:50:10 +00:00
}}.'
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh
2020-08-17 03:17:41 +00:00
summary : Processes experience elevated CPU throttling.
2020-06-08 17:34:52 +00:00
expr : |
sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace)
/
sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace)
> ( 25 / 100 )
for : 15m
labels :
2020-07-01 09:26:49 +00:00
severity : info
2020-06-08 17:34:52 +00:00
- name : kubernetes-storage
rules :
- alert : KubePersistentVolumeFillingUp
annotations :
2020-08-17 03:17:41 +00:00
description : The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
}} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage
2020-08-13 10:50:10 +00:00
}} free.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup
2020-08-17 03:17:41 +00:00
summary : PersistentVolume is filling up.
2020-06-08 17:34:52 +00:00
expr : |
2021-07-06 03:25:31 +00:00
(
kubelet_volume_stats_available_bytes{job="kubelet"}
/
kubelet_volume_stats_capacity_bytes{job="kubelet"}
) < 0.03
and
kubelet_volume_stats_used_bytes{job="kubelet"} > 0
2021-09-18 03:22:50 +00:00
unless on(namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
2020-06-08 17:34:52 +00:00
for : 1m
labels :
severity : critical
- alert : KubePersistentVolumeFillingUp
annotations :
2020-08-17 03:17:41 +00:00
description : Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim
2020-08-13 10:50:10 +00:00
}} in Namespace {{ $labels.namespace }} is expected to fill up within four
days. Currently {{ $value | humanizePercentage }} is available.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup
2020-08-17 03:17:41 +00:00
summary : PersistentVolume is filling up.
2020-06-08 17:34:52 +00:00
expr : |
(
kubelet_volume_stats_available_bytes{job="kubelet"}
/
kubelet_volume_stats_capacity_bytes{job="kubelet"}
) < 0.15
and
2021-07-06 03:25:31 +00:00
kubelet_volume_stats_used_bytes{job="kubelet"} > 0
and
2020-06-08 17:34:52 +00:00
predict_linear(kubelet_volume_stats_available_bytes{job="kubelet"}[6h], 4 * 24 * 3600) < 0
2021-09-18 03:22:50 +00:00
unless on(namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
2020-06-08 17:34:52 +00:00
for : 1h
labels :
severity : warning
- alert : KubePersistentVolumeErrors
annotations :
2020-08-17 03:17:41 +00:00
description : The persistent volume {{ $labels.persistentvolume }} has status
{{ $labels.phase }}.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors
2020-08-17 03:17:41 +00:00
summary : PersistentVolume is having issues with provisioning.
2020-06-08 17:34:52 +00:00
expr : |
kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
for : 5m
labels :
severity : critical
- name : kubernetes-system
rules :
- alert : KubeVersionMismatch
annotations :
2020-08-17 03:17:41 +00:00
description : There are {{ $value }} different semantic versions of Kubernetes
components running.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch
2020-08-17 03:17:41 +00:00
summary : Different semantic versions of Kubernetes components running.
2020-06-08 17:34:52 +00:00
expr : |
2021-02-05 03:25:52 +00:00
count(count by (git_version) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1
2020-06-08 17:34:52 +00:00
for : 15m
labels :
severity : warning
- alert : KubeClientErrors
annotations :
2020-08-17 03:17:41 +00:00
description : Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance
2020-08-13 10:50:10 +00:00
}}' is experiencing {{ $value | humanizePercentage }} errors.'
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors
2020-08-17 03:17:41 +00:00
summary : Kubernetes API server client is experiencing errors.
2020-06-08 17:34:52 +00:00
expr : |
2021-08-12 03:24:52 +00:00
(sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job, namespace)
2020-06-08 17:34:52 +00:00
/
2021-08-12 03:24:52 +00:00
sum(rate(rest_client_requests_total[5m])) by (instance, job, namespace))
2020-06-08 17:34:52 +00:00
> 0.01
for : 15m
labels :
severity : warning
- name : kube-apiserver-slos
rules :
- alert : KubeAPIErrorBudgetBurn
annotations :
2020-08-17 03:17:41 +00:00
description : The API server is burning too much error budget.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn
2020-08-17 03:17:41 +00:00
summary : The API server is burning too much error budget.
2020-06-08 17:34:52 +00:00
expr : |
sum(apiserver_request:burnrate1h) > (14.40 * 0.01000)
and
sum(apiserver_request:burnrate5m) > (14.40 * 0.01000)
for : 2m
labels :
long : 1h
severity : critical
short : 5m
- alert : KubeAPIErrorBudgetBurn
annotations :
2020-08-17 03:17:41 +00:00
description : The API server is burning too much error budget.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn
2020-08-17 03:17:41 +00:00
summary : The API server is burning too much error budget.
2020-06-08 17:34:52 +00:00
expr : |
sum(apiserver_request:burnrate6h) > (6.00 * 0.01000)
and
sum(apiserver_request:burnrate30m) > (6.00 * 0.01000)
for : 15m
labels :
long : 6h
severity : critical
short : 30m
- alert : KubeAPIErrorBudgetBurn
annotations :
2020-08-17 03:17:41 +00:00
description : The API server is burning too much error budget.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn
2020-08-17 03:17:41 +00:00
summary : The API server is burning too much error budget.
2020-06-08 17:34:52 +00:00
expr : |
sum(apiserver_request:burnrate1d) > (3.00 * 0.01000)
and
sum(apiserver_request:burnrate2h) > (3.00 * 0.01000)
for : 1h
labels :
long : 1d
severity : warning
short : 2h
- alert : KubeAPIErrorBudgetBurn
annotations :
2020-08-17 03:17:41 +00:00
description : The API server is burning too much error budget.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn
2020-08-17 03:17:41 +00:00
summary : The API server is burning too much error budget.
2020-06-08 17:34:52 +00:00
expr : |
sum(apiserver_request:burnrate3d) > (1.00 * 0.01000)
and
sum(apiserver_request:burnrate6h) > (1.00 * 0.01000)
for : 3h
labels :
long : 3d
severity : warning
short : 6h
- name : kubernetes-system-apiserver
rules :
- alert : KubeClientCertificateExpiration
annotations :
2020-08-17 03:17:41 +00:00
description : A client certificate used to authenticate to the apiserver is expiring
2020-08-13 10:50:10 +00:00
in less than 7.0 days.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration
2020-08-17 03:17:41 +00:00
summary : Client certificate is about to expire.
2020-06-08 17:34:52 +00:00
expr : |
apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 604800
labels :
severity : warning
- alert : KubeClientCertificateExpiration
annotations :
2020-08-17 03:17:41 +00:00
description : A client certificate used to authenticate to the apiserver is expiring
2020-08-13 10:50:10 +00:00
in less than 24.0 hours.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration
2020-08-17 03:17:41 +00:00
summary : Client certificate is about to expire.
2020-06-08 17:34:52 +00:00
expr : |
apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 86400
labels :
severity : critical
- alert : AggregatedAPIErrors
annotations :
2020-08-17 03:17:41 +00:00
description : An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has
2020-12-19 03:29:16 +00:00
reported errors. It has appeared unavailable {{ $value | humanize }} times
averaged over the past 10m.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapierrors
2020-08-17 03:17:41 +00:00
summary : An aggregated API has reported errors.
2020-06-08 17:34:52 +00:00
expr : |
2021-03-31 03:33:35 +00:00
sum by(name, namespace)(increase(aggregator_unavailable_apiservice_total[10m])) > 4
2020-06-08 17:34:52 +00:00
labels :
severity : warning
- alert : AggregatedAPIDown
annotations :
2020-08-17 03:17:41 +00:00
description : An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has
2020-08-26 03:18:43 +00:00
been only {{ $value | humanize }}% available over the last 10m.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapidown
2020-08-17 03:17:41 +00:00
summary : An aggregated API is down.
2020-06-08 17:34:52 +00:00
expr : |
2020-08-26 03:18:43 +00:00
(1 - max by(name, namespace)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85
2020-06-08 17:34:52 +00:00
for : 5m
labels :
severity : warning
- alert : KubeAPIDown
annotations :
2020-08-17 03:17:41 +00:00
description : KubeAPI has disappeared from Prometheus target discovery.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown
2020-08-17 03:17:41 +00:00
summary : Target disappeared from Prometheus target discovery.
2020-06-08 17:34:52 +00:00
expr : |
absent(up{job="kube-apiserver"} == 1)
for : 15m
labels :
severity : critical
2020-12-31 03:39:11 +00:00
- alert : KubeAPITerminatedRequests
annotations :
description : The apiserver has terminated {{ $value | humanizePercentage }}
of its incoming requests.
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapiterminatedrequests
summary : The apiserver has terminated {{ $value | humanizePercentage }} of its
incoming requests.
expr : |
sum(rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="kube-apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m])) ) > 0.20
for : 5m
labels :
severity : warning
2020-06-08 17:34:52 +00:00
- name : kubernetes-system-kubelet
rules :
- alert : KubeNodeNotReady
annotations :
2020-08-17 03:17:41 +00:00
description : '{{ $labels.node }} has been unready for more than 15 minutes.'
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready
2020-08-17 03:17:41 +00:00
summary : Node is not ready.
2020-06-08 17:34:52 +00:00
expr : |
kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0
for : 15m
labels :
severity : warning
- alert : KubeNodeUnreachable
annotations :
2020-08-17 03:17:41 +00:00
description : '{{ $labels.node }} is unreachable and some workloads may be rescheduled.'
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodeunreachable
2020-08-17 03:17:41 +00:00
summary : Node is unreachable.
2020-06-08 17:34:52 +00:00
expr : |
2020-07-20 14:43:19 +00:00
(kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1
2020-08-26 03:18:43 +00:00
for : 15m
2020-06-08 17:34:52 +00:00
labels :
severity : warning
- alert : KubeletTooManyPods
annotations :
2020-08-17 03:17:41 +00:00
description : Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage
2020-08-13 10:50:10 +00:00
}} of its Pod capacity.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods
2020-08-17 03:17:41 +00:00
summary : Kubelet is running at capacity.
2020-06-08 17:34:52 +00:00
expr : |
2020-07-03 12:42:21 +00:00
count by(node) (
(kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"})
)
/
max by(node) (
2021-03-26 04:18:04 +00:00
kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1
2020-07-03 12:42:21 +00:00
) > 0.95
2020-06-08 17:34:52 +00:00
for : 15m
labels :
2021-08-24 03:23:02 +00:00
severity : info
2020-06-08 17:34:52 +00:00
- alert : KubeNodeReadinessFlapping
annotations :
2020-08-17 03:17:41 +00:00
description : The readiness status of node {{ $labels.node }} has changed {{
$value }} times in the last 15 minutes.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodereadinessflapping
2020-08-17 03:17:41 +00:00
summary : Node readiness status is flapping.
2020-06-08 17:34:52 +00:00
expr : |
sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (node) > 2
for : 15m
labels :
severity : warning
- alert : KubeletPlegDurationHigh
annotations :
2020-08-17 03:17:41 +00:00
description : The Kubelet Pod Lifecycle Event Generator has a 99th percentile
duration of {{ $value }} seconds on node {{ $labels.node }}.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletplegdurationhigh
2020-08-17 03:17:41 +00:00
summary : Kubelet Pod Lifecycle Event Generator is taking too long to relist.
2020-06-08 17:34:52 +00:00
expr : |
node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10
for : 5m
labels :
severity : warning
- alert : KubeletPodStartUpLatencyHigh
annotations :
2020-08-17 03:17:41 +00:00
description : Kubelet Pod startup 99th percentile latency is {{ $value }} seconds
2020-08-13 10:50:10 +00:00
on node {{ $labels.node }}.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletpodstartuplatencyhigh
2020-08-17 03:17:41 +00:00
summary : Kubelet Pod startup latency is too high.
2020-06-08 17:34:52 +00:00
expr : |
histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job="kubelet"} > 60
for : 15m
labels :
severity : warning
2020-08-27 03:18:25 +00:00
- alert : KubeletClientCertificateExpiration
annotations :
description : Client certificate for Kubelet on node {{ $labels.node }} expires
in {{ $value | humanizeDuration }}.
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletclientcertificateexpiration
summary : Kubelet client certificate is about to expire.
expr : |
kubelet_certificate_manager_client_ttl_seconds < 604800
labels :
severity : warning
- alert : KubeletClientCertificateExpiration
annotations :
description : Client certificate for Kubelet on node {{ $labels.node }} expires
in {{ $value | humanizeDuration }}.
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletclientcertificateexpiration
summary : Kubelet client certificate is about to expire.
expr : |
kubelet_certificate_manager_client_ttl_seconds < 86400
labels :
severity : critical
- alert : KubeletServerCertificateExpiration
annotations :
description : Server certificate for Kubelet on node {{ $labels.node }} expires
in {{ $value | humanizeDuration }}.
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletservercertificateexpiration
summary : Kubelet server certificate is about to expire.
expr : |
kubelet_certificate_manager_server_ttl_seconds < 604800
labels :
severity : warning
- alert : KubeletServerCertificateExpiration
annotations :
description : Server certificate for Kubelet on node {{ $labels.node }} expires
in {{ $value | humanizeDuration }}.
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletservercertificateexpiration
summary : Kubelet server certificate is about to expire.
expr : |
kubelet_certificate_manager_server_ttl_seconds < 86400
labels :
severity : critical
- alert : KubeletClientCertificateRenewalErrors
annotations :
description : Kubelet on node {{ $labels.node }} has failed to renew its client
certificate ({{ $value | humanize }} errors in the last 5 minutes).
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletclientcertificaterenewalerrors
summary : Kubelet has failed to renew its client certificate.
expr : |
increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0
for : 15m
labels :
severity : warning
- alert : KubeletServerCertificateRenewalErrors
annotations :
description : Kubelet on node {{ $labels.node }} has failed to renew its server
certificate ({{ $value | humanize }} errors in the last 5 minutes).
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletservercertificaterenewalerrors
summary : Kubelet has failed to renew its server certificate.
expr : |
increase(kubelet_server_expiration_renew_errors[5m]) > 0
for : 15m
labels :
severity : warning
2020-06-08 17:34:52 +00:00
- alert : KubeletDown
annotations :
2020-08-17 03:17:41 +00:00
description : Kubelet has disappeared from Prometheus target discovery.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown
2020-08-17 03:17:41 +00:00
summary : Target disappeared from Prometheus target discovery.
2020-06-08 17:34:52 +00:00
expr : |
absent(up{job="kubelet"} == 1)
for : 15m
labels :
severity : critical
- name : kubernetes-system-scheduler
rules :
- alert : KubeSchedulerDown
annotations :
2020-08-17 03:17:41 +00:00
description : KubeScheduler has disappeared from Prometheus target discovery.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown
2020-08-17 03:17:41 +00:00
summary : Target disappeared from Prometheus target discovery.
2020-06-08 17:34:52 +00:00
expr : |
absent(up{job="kube-scheduler"} == 1)
for : 15m
labels :
severity : critical
- name : kubernetes-system-controller-manager
rules :
- alert : KubeControllerManagerDown
annotations :
2020-08-17 03:17:41 +00:00
description : KubeControllerManager has disappeared from Prometheus target discovery.
2020-06-08 17:34:52 +00:00
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown
2020-08-17 03:17:41 +00:00
summary : Target disappeared from Prometheus target discovery.
2020-06-08 17:34:52 +00:00
expr : |
absent(up{job="kube-controller-manager"} == 1)
for : 15m
labels :
severity : critical
2021-10-15 03:23:48 +00:00
- name : kubernetes-system-kube-proxy
rules :
- alert : KubeProxyDown
annotations :
description : KubeProxy has disappeared from Prometheus target discovery.
runbook_url : https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeproxydown
summary : Target disappeared from Prometheus target discovery.
expr : |
absent(up{job="kube-proxy"} == 1)
for : 15m
labels :
severity : critical