monitoring-mixins-website/assets/kubernetes/alerts.yaml

groups:
- name: kubernetes-apps
  rules:
  - alert: KubePodCrashLooping
    annotations:
      message: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting {{ printf "%.2f" $value }} times / 5 minutes.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping
    expr: |
      rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[5m]) * 60 * 5 > 0
    for: 15m
    labels:
      severity: warning
  - alert: KubePodNotReady
    annotations:
      message: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready
    expr: |
      sum by (namespace, pod) (
        max by(namespace, pod) (
          kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown"}
        ) * on(namespace, pod) group_left(owner_kind) topk by(namespace, pod) (
          1, max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"})
        )
      ) > 0
    for: 15m
    labels:
      severity: warning
  - alert: KubeDeploymentGenerationMismatch
    annotations:
      message: Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch
    expr: |
      kube_deployment_status_observed_generation{job="kube-state-metrics"}
        !=
      kube_deployment_metadata_generation{job="kube-state-metrics"}
    for: 15m
    labels:
      severity: warning
  - alert: KubeDeploymentReplicasMismatch
    annotations:
      message: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch
    expr: |
      (
        kube_deployment_spec_replicas{job="kube-state-metrics"}
          !=
        kube_deployment_status_replicas_available{job="kube-state-metrics"}
      ) and (
        changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[5m])
          ==
        0
      )
    for: 15m
    labels:
      severity: warning
  - alert: KubeStatefulSetReplicasMismatch
    annotations:
      message: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch
    expr: |
      (
        kube_statefulset_status_replicas_ready{job="kube-state-metrics"}
          !=
        kube_statefulset_status_replicas{job="kube-state-metrics"}
      ) and (
        changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m])
          ==
        0
      )
    for: 15m
    labels:
      severity: warning
  - alert: KubeStatefulSetGenerationMismatch
    annotations:
      message: StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch
    expr: |
      kube_statefulset_status_observed_generation{job="kube-state-metrics"}
        !=
      kube_statefulset_metadata_generation{job="kube-state-metrics"}
    for: 15m
    labels:
      severity: warning
  - alert: KubeStatefulSetUpdateNotRolledOut
    annotations:
      message: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout
    expr: |
      (
        max without (revision) (
          kube_statefulset_status_current_revision{job="kube-state-metrics"}
            unless
          kube_statefulset_status_update_revision{job="kube-state-metrics"}
        )
          *
        (
          kube_statefulset_replicas{job="kube-state-metrics"}
            !=
          kube_statefulset_status_replicas_updated{job="kube-state-metrics"}
        )
      )  and (
        changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m])
          ==
        0
      )
    for: 15m
    labels:
      severity: warning
  - alert: KubeDaemonSetRolloutStuck
    annotations:
      message: Only {{ $value | humanizePercentage }} of the desired Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are scheduled and ready.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck
    expr: |
      kube_daemonset_status_number_ready{job="kube-state-metrics"}
        /
      kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} < 1.00
    for: 15m
    labels:
      severity: warning
  - alert: KubeContainerWaiting
    annotations:
      message: Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} has been in waiting state for longer than 1 hour.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting
    expr: |
      sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0
    for: 1h
    labels:
      severity: warning
  - alert: KubeDaemonSetNotScheduled
    annotations:
      message: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.'
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled
    expr: |
      kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
        -
      kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0
    for: 10m
    labels:
      severity: warning
  - alert: KubeDaemonSetMisScheduled
    annotations:
      message: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.'
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled
    expr: |
      kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0
    for: 15m
    labels:
      severity: warning
  - alert: KubeJobCompletion
    annotations:
      message: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than one hour to complete.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion
    expr: |
      kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"}  > 0
    for: 1h
    labels:
      severity: warning
  - alert: KubeJobFailed
    annotations:
      message: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed
    expr: |
      kube_job_failed{job="kube-state-metrics"}  > 0
    for: 15m
    labels:
      severity: warning
  - alert: KubeHpaReplicasMismatch
    annotations:
      message: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has not matched the desired number of replicas for longer than 15 minutes.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch
    expr: |
      (kube_hpa_status_desired_replicas{job="kube-state-metrics"}
        !=
      kube_hpa_status_current_replicas{job="kube-state-metrics"})
        and
      changes(kube_hpa_status_current_replicas[15m]) == 0
    for: 15m
    labels:
      severity: warning
  - alert: KubeHpaMaxedOut
    annotations:
      message: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has been running at max replicas for longer than 15 minutes.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout
    expr: |
      kube_hpa_status_current_replicas{job="kube-state-metrics"}
        ==
      kube_hpa_spec_max_replicas{job="kube-state-metrics"}
    for: 15m
    labels:
      severity: warning
- name: kubernetes-resources
  rules:
  - alert: KubeCPUOvercommit
    annotations:
      message: Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit
    expr: |
      sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum{})
        /
      sum(kube_node_status_allocatable_cpu_cores)
        >
      (count(kube_node_status_allocatable_cpu_cores)-1) / count(kube_node_status_allocatable_cpu_cores)
    for: 5m
    labels:
      severity: warning
  - alert: KubeMemoryOvercommit
    annotations:
      message: Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryovercommit
    expr: |
      sum(namespace:kube_pod_container_resource_requests_memory_bytes:sum{})
        /
      sum(kube_node_status_allocatable_memory_bytes)
        >
      (count(kube_node_status_allocatable_memory_bytes)-1)
        /
      count(kube_node_status_allocatable_memory_bytes)
    for: 5m
    labels:
      severity: warning
  - alert: KubeCPUQuotaOvercommit
    annotations:
      message: Cluster has overcommitted CPU resource requests for Namespaces.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuquotaovercommit
    expr: |
      sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"})
        /
      sum(kube_node_status_allocatable_cpu_cores)
        > 1.5
    for: 5m
    labels:
      severity: warning
  - alert: KubeMemoryQuotaOvercommit
    annotations:
      message: Cluster has overcommitted memory resource requests for Namespaces.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryquotaovercommit
    expr: |
      sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"})
        /
      sum(kube_node_status_allocatable_memory_bytes{job="node-exporter"})
        > 1.5
    for: 5m
    labels:
      severity: warning
  - alert: KubeQuotaFullyUsed
    annotations:
      message: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotafullyused
    expr: |
      kube_resourcequota{job="kube-state-metrics", type="used"}
        / ignoring(instance, job, type)
      (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
        >= 1
    for: 15m
    labels:
      severity: info
  - alert: CPUThrottlingHigh
    annotations:
      message: '{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.'
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh
    expr: |
      sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace)
        /
      sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace)
        > ( 25 / 100 )
    for: 15m
    labels:
      severity: info
- name: kubernetes-storage
  rules:
  - alert: KubePersistentVolumeFillingUp
    annotations:
      message: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup
    expr: |
      kubelet_volume_stats_available_bytes{job="kubelet"}
        /
      kubelet_volume_stats_capacity_bytes{job="kubelet"}
        < 0.03
    for: 1m
    labels:
      severity: critical
  - alert: KubePersistentVolumeFillingUp
    annotations:
      message: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup
    expr: |
      (
        kubelet_volume_stats_available_bytes{job="kubelet"}
          /
        kubelet_volume_stats_capacity_bytes{job="kubelet"}
      ) < 0.15
      and
      predict_linear(kubelet_volume_stats_available_bytes{job="kubelet"}[6h], 4 * 24 * 3600) < 0
    for: 1h
    labels:
      severity: warning
  - alert: KubePersistentVolumeErrors
    annotations:
      message: The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors
    expr: |
      kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
    for: 5m
    labels:
      severity: critical
- name: kubernetes-system
  rules:
  - alert: KubeVersionMismatch
    annotations:
      message: There are {{ $value }} different semantic versions of Kubernetes components running.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch
    expr: |
      count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"gitVersion","$1","gitVersion","(v[0-9]*.[0-9]*).*"))) > 1
    for: 15m
    labels:
      severity: warning
  - alert: KubeClientErrors
    annotations:
      message: Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors
    expr: |
      (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job)
        /
      sum(rate(rest_client_requests_total[5m])) by (instance, job))
      > 0.01
    for: 15m
    labels:
      severity: warning
- name: kube-apiserver-slos
  rules:
  - alert: KubeAPIErrorBudgetBurn
    annotations:
      message: The API server is burning too much error budget
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn
    expr: |
      sum(apiserver_request:burnrate1h) > (14.40 * 0.01000)
      and
      sum(apiserver_request:burnrate5m) > (14.40 * 0.01000)
    for: 2m
    labels:
      long: 1h
      severity: critical
      short: 5m
  - alert: KubeAPIErrorBudgetBurn
    annotations:
      message: The API server is burning too much error budget
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn
    expr: |
      sum(apiserver_request:burnrate6h) > (6.00 * 0.01000)
      and
      sum(apiserver_request:burnrate30m) > (6.00 * 0.01000)
    for: 15m
    labels:
      long: 6h
      severity: critical
      short: 30m
  - alert: KubeAPIErrorBudgetBurn
    annotations:
      message: The API server is burning too much error budget
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn
    expr: |
      sum(apiserver_request:burnrate1d) > (3.00 * 0.01000)
      and
      sum(apiserver_request:burnrate2h) > (3.00 * 0.01000)
    for: 1h
    labels:
      long: 1d
      severity: warning
      short: 2h
  - alert: KubeAPIErrorBudgetBurn
    annotations:
      message: The API server is burning too much error budget
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn
    expr: |
      sum(apiserver_request:burnrate3d) > (1.00 * 0.01000)
      and
      sum(apiserver_request:burnrate6h) > (1.00 * 0.01000)
    for: 3h
    labels:
      long: 3d
      severity: warning
      short: 6h
- name: kubernetes-system-apiserver
  rules:
  - alert: KubeClientCertificateExpiration
    annotations:
      message: A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration
    expr: |
      apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 604800
    labels:
      severity: warning
  - alert: KubeClientCertificateExpiration
    annotations:
      message: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration
    expr: |
      apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 86400
    labels:
      severity: critical
  - alert: AggregatedAPIErrors
    annotations:
      message: An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. The number of errors have increased for it in the past five minutes. High values indicate that the availability of the service changes too often.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapierrors
    expr: |
      sum by(name, namespace)(increase(aggregator_unavailable_apiservice_count[5m])) > 2
    labels:
      severity: warning
  - alert: AggregatedAPIDown
    annotations:
      message: An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 5m.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapidown
    expr: |
      (1 - max by(name, namespace)(avg_over_time(aggregator_unavailable_apiservice[5m]))) * 100 < 90
    for: 5m
    labels:
      severity: warning
  - alert: KubeAPIDown
    annotations:
      message: KubeAPI has disappeared from Prometheus target discovery.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown
    expr: |
      absent(up{job="kube-apiserver"} == 1)
    for: 15m
    labels:
      severity: critical
- name: kubernetes-system-kubelet
  rules:
  - alert: KubeNodeNotReady
    annotations:
      message: '{{ $labels.node }} has been unready for more than 15 minutes.'
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready
    expr: |
      kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0
    for: 15m
    labels:
      severity: warning
  - alert: KubeNodeUnreachable
    annotations:
      message: '{{ $labels.node }} is unreachable and some workloads may be rescheduled.'
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodeunreachable
    expr: |
      (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1
    labels:
      severity: warning
  - alert: KubeletTooManyPods
    annotations:
      message: Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods
    expr: |
      count by(node) (
        (kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"})
      )
      /
      max by(node) (
        kube_node_status_capacity_pods{job="kube-state-metrics"} != 1
      ) > 0.95
    for: 15m
    labels:
      severity: warning
  - alert: KubeNodeReadinessFlapping
    annotations:
      message: The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodereadinessflapping
    expr: |
      sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (node) > 2
    for: 15m
    labels:
      severity: warning
  - alert: KubeletPlegDurationHigh
    annotations:
      message: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletplegdurationhigh
    expr: |
      node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10
    for: 5m
    labels:
      severity: warning
  - alert: KubeletPodStartUpLatencyHigh
    annotations:
      message: Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletpodstartuplatencyhigh
    expr: |
      histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job="kubelet"} > 60
    for: 15m
    labels:
      severity: warning
  - alert: KubeletDown
    annotations:
      message: Kubelet has disappeared from Prometheus target discovery.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown
    expr: |
      absent(up{job="kubelet"} == 1)
    for: 15m
    labels:
      severity: critical
- name: kubernetes-system-scheduler
  rules:
  - alert: KubeSchedulerDown
    annotations:
      message: KubeScheduler has disappeared from Prometheus target discovery.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown
    expr: |
      absent(up{job="kube-scheduler"} == 1)
    for: 15m
    labels:
      severity: critical
- name: kubernetes-system-controller-manager
  rules:
  - alert: KubeControllerManagerDown
    annotations:
      message: KubeControllerManager has disappeared from Prometheus target discovery.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown
    expr: |
      absent(up{job="kube-controller-manager"} == 1)
    for: 15m
    labels:
      severity: critical