monitoring-mixins-website/assets/prometheus/alerts.yaml

groups:
- name: prometheus
  rules:
  - alert: PrometheusBadConfig
    annotations:
      description: Prometheus {{$labels.instance}} has failed to reload its configuration.
      summary: Failed Prometheus configuration reload.
    expr: |
      # Without max_over_time, failed scrapes could create false negatives, see
      # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
      max_over_time(prometheus_config_last_reload_successful{job="prometheus"}[5m]) == 0      
    for: 10m
    labels:
      severity: critical
  - alert: PrometheusSDRefreshFailure
    annotations:
      description: Prometheus {{$labels.instance}} has failed to refresh SD with mechanism
        {{$labels.mechanism}}.
      summary: Failed Prometheus SD refresh.
    expr: |
      increase(prometheus_sd_refresh_failures_total{job="prometheus"}[10m]) > 0      
    for: 20m
    labels:
      severity: warning
  - alert: PrometheusKubernetesListWatchFailures
    annotations:
      description: Kubernetes service discovery of Prometheus {{$labels.instance}}
        is experiencing {{ printf "%.0f" $value }} failures with LIST/WATCH requests
        to the Kubernetes API in the last 5 minutes.
      summary: Requests in Kubernetes SD are failing.
    expr: |
      increase(prometheus_sd_kubernetes_failures_total{job="prometheus"}[5m]) > 0      
    for: 15m
    labels:
      severity: warning
  - alert: PrometheusNotificationQueueRunningFull
    annotations:
      description: Alert notification queue of Prometheus {{$labels.instance}} is
        running full.
      summary: Prometheus alert notification queue predicted to run full in less than
        30m.
    expr: |
      # Without min_over_time, failed scrapes could create false negatives, see
      # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
      (
        predict_linear(prometheus_notifications_queue_length{job="prometheus"}[5m], 60 * 30)
      >
        min_over_time(prometheus_notifications_queue_capacity{job="prometheus"}[5m])
      )      
    for: 15m
    labels:
      severity: warning
  - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers
    annotations:
      description: '{{ printf "%.1f" $value }}% errors while sending alerts from Prometheus
        {{$labels.instance}} to Alertmanager {{$labels.alertmanager}}.'
      summary: Prometheus has encountered more than 1% errors sending alerts to a
        specific Alertmanager.
    expr: |
      (
        rate(prometheus_notifications_errors_total{job="prometheus"}[5m])
      /
        rate(prometheus_notifications_sent_total{job="prometheus"}[5m])
      )
      * 100
      > 1      
    for: 15m
    labels:
      severity: warning
  - alert: PrometheusNotConnectedToAlertmanagers
    annotations:
      description: Prometheus {{$labels.instance}} is not connected to any Alertmanagers.
      summary: Prometheus is not connected to any Alertmanagers.
    expr: |
      # Without max_over_time, failed scrapes could create false negatives, see
      # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
      max_over_time(prometheus_notifications_alertmanagers_discovered{job="prometheus"}[5m]) < 1      
    for: 10m
    labels:
      severity: warning
  - alert: PrometheusTSDBReloadsFailing
    annotations:
      description: Prometheus {{$labels.instance}} has detected {{$value | humanize}}
        reload failures over the last 3h.
      summary: Prometheus has issues reloading blocks from disk.
    expr: |
      increase(prometheus_tsdb_reloads_failures_total{job="prometheus"}[3h]) > 0      
    for: 4h
    labels:
      severity: warning
  - alert: PrometheusTSDBCompactionsFailing
    annotations:
      description: Prometheus {{$labels.instance}} has detected {{$value | humanize}}
        compaction failures over the last 3h.
      summary: Prometheus has issues compacting blocks.
    expr: |
      increase(prometheus_tsdb_compactions_failed_total{job="prometheus"}[3h]) > 0      
    for: 4h
    labels:
      severity: warning
  - alert: PrometheusNotIngestingSamples
    annotations:
      description: Prometheus {{$labels.instance}} is not ingesting samples.
      summary: Prometheus is not ingesting samples.
    expr: |
      (
        sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="prometheus"}[5m])) <= 0
      and
        (
          sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="prometheus"}) > 0
        or
          sum without(rule_group) (prometheus_rule_group_rules{job="prometheus"}) > 0
        )
      )      
    for: 10m
    labels:
      severity: warning
  - alert: PrometheusDuplicateTimestamps
    annotations:
      description: Prometheus {{$labels.instance}} is dropping {{ printf "%.4g" $value  }}
        samples/s with different values but duplicated timestamp.
      summary: Prometheus is dropping samples with duplicate timestamps.
    expr: |
      rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus"}[5m]) > 0      
    for: 10m
    labels:
      severity: warning
  - alert: PrometheusOutOfOrderTimestamps
    annotations:
      description: Prometheus {{$labels.instance}} is dropping {{ printf "%.4g" $value  }}
        samples/s with timestamps arriving out of order.
      summary: Prometheus drops samples with out-of-order timestamps.
    expr: |
      rate(prometheus_target_scrapes_sample_out_of_order_total{job="prometheus"}[5m]) > 0      
    for: 10m
    labels:
      severity: warning
  - alert: PrometheusRemoteStorageFailures
    annotations:
      description: Prometheus {{$labels.instance}} failed to send {{ printf "%.1f"
        $value }}% of the samples to {{ $labels.remote_name}}:{{ $labels.url }}
      summary: Prometheus fails to send samples to remote storage.
    expr: |
      (
        (rate(prometheus_remote_storage_failed_samples_total{job="prometheus"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus"}[5m]))
      /
        (
          (rate(prometheus_remote_storage_failed_samples_total{job="prometheus"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus"}[5m]))
        +
          (rate(prometheus_remote_storage_succeeded_samples_total{job="prometheus"}[5m]) or rate(prometheus_remote_storage_samples_total{job="prometheus"}[5m]))
        )
      )
      * 100
      > 1      
    for: 15m
    labels:
      severity: critical
  - alert: PrometheusRemoteWriteBehind
    annotations:
      description: Prometheus {{$labels.instance}} remote write is {{ printf "%.1f"
        $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url }}.
      summary: Prometheus remote write is behind.
    expr: |
      # Without max_over_time, failed scrapes could create false negatives, see
      # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
      (
        max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="prometheus"}[5m])
      - ignoring(remote_name, url) group_right
        max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="prometheus"}[5m])
      )
      > 120      
    for: 15m
    labels:
      severity: critical
  - alert: PrometheusRemoteWriteDesiredShards
    annotations:
      description: Prometheus {{$labels.instance}} remote write desired shards calculation
        wants to run {{ $value }} shards for queue {{ $labels.remote_name}}:{{ $labels.url
        }}, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance="%s",job="prometheus"}`
        $labels.instance | query | first | value }}.
      summary: Prometheus remote write desired shards calculation wants to run more
        than configured max shards.
    expr: |
      # Without max_over_time, failed scrapes could create false negatives, see
      # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
      (
        max_over_time(prometheus_remote_storage_shards_desired{job="prometheus"}[5m])
      >
        max_over_time(prometheus_remote_storage_shards_max{job="prometheus"}[5m])
      )      
    for: 15m
    labels:
      severity: warning
  - alert: PrometheusRuleFailures
    annotations:
      description: Prometheus {{$labels.instance}} has failed to evaluate {{ printf
        "%.0f" $value }} rules in the last 5m.
      summary: Prometheus is failing rule evaluations.
    expr: |
      increase(prometheus_rule_evaluation_failures_total{job="prometheus"}[5m]) > 0      
    for: 15m
    labels:
      severity: critical
  - alert: PrometheusMissingRuleEvaluations
    annotations:
      description: Prometheus {{$labels.instance}} has missed {{ printf "%.0f" $value
        }} rule group evaluations in the last 5m.
      summary: Prometheus is missing rule evaluations due to slow rule group evaluation.
    expr: |
      increase(prometheus_rule_group_iterations_missed_total{job="prometheus"}[5m]) > 0      
    for: 15m
    labels:
      severity: warning
  - alert: PrometheusTargetLimitHit
    annotations:
      description: Prometheus {{$labels.instance}} has dropped {{ printf "%.0f" $value
        }} targets because the number of targets exceeded the configured target_limit.
      summary: Prometheus has dropped targets because some scrape configs have exceeded
        the targets limit.
    expr: |
      increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="prometheus"}[5m]) > 0      
    for: 15m
    labels:
      severity: warning
  - alert: PrometheusLabelLimitHit
    annotations:
      description: Prometheus {{$labels.instance}} has dropped {{ printf "%.0f" $value
        }} targets because some samples exceeded the configured label_limit, label_name_length_limit
        or label_value_length_limit.
      summary: Prometheus has dropped targets because some scrape configs have exceeded
        the labels limit.
    expr: |
      increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="prometheus"}[5m]) > 0      
    for: 15m
    labels:
      severity: warning
  - alert: PrometheusScrapeBodySizeLimitHit
    annotations:
      description: Prometheus {{$labels.instance}} has failed {{ printf "%.0f" $value
        }} scrapes in the last 5m because some targets exceeded the configured body_size_limit.
      summary: Prometheus has dropped some targets that exceeded body size limit.
    expr: |
      increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="prometheus"}[5m]) > 0      
    for: 15m
    labels:
      severity: warning
  - alert: PrometheusScrapeSampleLimitHit
    annotations:
      description: Prometheus {{$labels.instance}} has failed {{ printf "%.0f" $value
        }} scrapes in the last 5m because some targets exceeded the configured sample_limit.
      summary: Prometheus has failed scrapes that have exceeded the configured sample
        limit.
    expr: |
      increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="prometheus"}[5m]) > 0      
    for: 15m
    labels:
      severity: warning
  - alert: PrometheusTargetSyncFailure
    annotations:
      description: '{{ printf "%.0f" $value }} targets in Prometheus {{$labels.instance}}
        have failed to sync because invalid configuration was supplied.'
      summary: Prometheus has failed to sync targets.
    expr: |
      increase(prometheus_target_sync_failed_total{job="prometheus"}[30m]) > 0      
    for: 5m
    labels:
      severity: critical
  - alert: PrometheusHighQueryLoad
    annotations:
      description: Prometheus {{$labels.instance}} query API has less than 20% available
        capacity in its query engine for the last 15 minutes.
      summary: Prometheus is reaching its maximum capacity serving concurrent requests.
    expr: |
      avg_over_time(prometheus_engine_queries{job="prometheus"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="prometheus"}[5m]) > 0.8      
    for: 15m
    labels:
      severity: warning
  - alert: PrometheusErrorSendingAlertsToAnyAlertmanager
    annotations:
      description: '{{ printf "%.1f" $value }}% minimum errors while sending alerts
        from Prometheus {{$labels.instance}} to any Alertmanager.'
      summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager.
    expr: |
      min without (alertmanager) (
        rate(prometheus_notifications_errors_total{job="prometheus",alertmanager!~``}[5m])
      /
        rate(prometheus_notifications_sent_total{job="prometheus",alertmanager!~``}[5m])
      )
      * 100
      > 3      
    for: 15m
    labels:
      severity: critical