diff --git a/assets/prometheus/alerts.yaml b/assets/prometheus/alerts.yaml index ffbff5b..3f7f61b 100644 --- a/assets/prometheus/alerts.yaml +++ b/assets/prometheus/alerts.yaml @@ -213,6 +213,16 @@ groups: for: 15m labels: severity: warning + - alert: PrometheusTargetSyncFailure + annotations: + description: '{{ printf "%.0f" $value }} targets in Prometheus {{$labels.instance}} + have failed to sync because invalid configuration was supplied.' + summary: Prometheus has failed to sync targets. + expr: | + increase(prometheus_target_sync_failed_total{job="prometheus"}[30m]) > 0 + for: 5m + labels: + severity: critical - alert: PrometheusErrorSendingAlertsToAnyAlertmanager annotations: description: '{{ printf "%.1f" $value }}% minimum errors while sending alerts diff --git a/site/content/prometheus/_index.md b/site/content/prometheus/_index.md index e178801..32cd3ef 100644 --- a/site/content/prometheus/_index.md +++ b/site/content/prometheus/_index.md @@ -314,6 +314,21 @@ labels: severity: warning {{< /code >}} +##### PrometheusTargetSyncFailure + +{{< code lang="yaml" >}} +alert: PrometheusTargetSyncFailure +annotations: + description: '{{ printf "%.0f" $value }} targets in Prometheus {{$labels.instance}} + have failed to sync because invalid configuration was supplied.' + summary: Prometheus has failed to sync targets. +expr: | + increase(prometheus_target_sync_failed_total{job="prometheus"}[30m]) > 0 +for: 5m +labels: + severity: critical +{{< /code >}} + ##### PrometheusErrorSendingAlertsToAnyAlertmanager '{{ printf "%.1f" $value }}% minimum errors while sending alerts from Prometheus encounters more than 3% errors sending alerts to any Alertmanager.