2020-07-01 09:17:43 +00:00
|
|
|
groups:
|
|
|
|
- name: cortex_alerts
|
|
|
|
rules:
|
|
|
|
- alert: CortexIngesterUnhealthy
|
|
|
|
annotations:
|
|
|
|
message: There are {{ printf "%f" $value }} unhealthy ingester(s).
|
|
|
|
expr: |
|
|
|
|
min by (cluster, namespace) (cortex_ring_members{state="Unhealthy", name="ingester"}) > 0
|
|
|
|
for: 15m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexRequestErrors
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
{{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors.
|
|
|
|
expr: |
|
2020-12-15 03:27:30 +00:00
|
|
|
100 * sum by (cluster, namespace, job, route) (rate(cortex_request_duration_seconds_count{status_code=~"5..",route!~"ready"}[1m]))
|
2020-07-01 09:17:43 +00:00
|
|
|
/
|
2020-12-15 03:27:30 +00:00
|
|
|
sum by (cluster, namespace, job, route) (rate(cortex_request_duration_seconds_count{route!~"ready"}[1m]))
|
2020-07-01 09:17:43 +00:00
|
|
|
> 1
|
|
|
|
for: 15m
|
|
|
|
labels:
|
|
|
|
severity: warning
|
|
|
|
- alert: CortexRequestLatency
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
{{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency.
|
|
|
|
expr: |
|
2021-01-07 03:47:04 +00:00
|
|
|
cluster_namespace_job_route:cortex_request_duration_seconds:99quantile{route!~"metrics|/frontend.Frontend/Process|ready|/schedulerpb.SchedulerForFrontend/FrontendLoop|/schedulerpb.SchedulerForQuerier/QuerierLoop"}
|
2020-07-01 09:17:43 +00:00
|
|
|
>
|
|
|
|
2.5
|
|
|
|
for: 15m
|
|
|
|
labels:
|
|
|
|
severity: warning
|
|
|
|
- alert: CortexTableSyncFailure
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
{{ $labels.job }} is experiencing {{ printf "%.2f" $value }}% errors syncing tables.
|
|
|
|
expr: |
|
|
|
|
100 * rate(cortex_table_manager_sync_duration_seconds_count{status_code!~"2.."}[15m])
|
|
|
|
/
|
|
|
|
rate(cortex_table_manager_sync_duration_seconds_count[15m])
|
|
|
|
> 10
|
|
|
|
for: 30m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexQueriesIncorrect
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
Incorrect results for {{ printf "%.2f" $value }}% of queries.
|
|
|
|
expr: |
|
|
|
|
100 * sum by (cluster, namespace) (rate(test_exporter_test_case_result_total{result="fail"}[5m]))
|
|
|
|
/
|
|
|
|
sum by (cluster, namespace) (rate(test_exporter_test_case_result_total[5m])) > 1
|
|
|
|
for: 15m
|
|
|
|
labels:
|
|
|
|
severity: warning
|
2020-08-12 17:33:36 +00:00
|
|
|
- alert: CortexInconsistentConfig
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
An inconsistent config file hash is used across cluster {{ $labels.job }}.
|
|
|
|
expr: |
|
|
|
|
count(count by(cluster, namespace, job, sha256) (cortex_config_hash)) without(sha256) > 1
|
|
|
|
for: 1h
|
|
|
|
labels:
|
|
|
|
severity: warning
|
2020-07-20 14:43:19 +00:00
|
|
|
- alert: CortexBadRuntimeConfig
|
2020-07-01 09:17:43 +00:00
|
|
|
annotations:
|
|
|
|
message: |
|
2020-07-20 14:43:19 +00:00
|
|
|
{{ $labels.job }} failed to reload runtime config.
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
2020-07-20 14:43:19 +00:00
|
|
|
cortex_runtime_config_last_reload_successful == 0
|
|
|
|
or
|
2020-07-01 09:17:43 +00:00
|
|
|
cortex_overrides_last_reload_successful == 0
|
2021-01-19 04:04:52 +00:00
|
|
|
for: 5m
|
2020-07-01 09:17:43 +00:00
|
|
|
labels:
|
|
|
|
severity: warning
|
|
|
|
- alert: CortexQuerierCapacityFull
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
{{ $labels.job }} is at capacity processing queries.
|
|
|
|
expr: |
|
|
|
|
prometheus_engine_queries_concurrent_max{job=~".+/(cortex|ruler|querier)"} - prometheus_engine_queries{job=~".+/(cortex|ruler|querier)"} == 0
|
|
|
|
for: 5m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexFrontendQueriesStuck
|
|
|
|
annotations:
|
|
|
|
message: |
|
2020-12-15 03:27:30 +00:00
|
|
|
There are {{ $value }} queued up queries in query-frontend.
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
|
|
|
sum by (cluster, namespace) (cortex_query_frontend_queue_length) > 1
|
|
|
|
for: 5m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
2020-12-15 03:27:30 +00:00
|
|
|
- alert: CortexSchedulerQueriesStuck
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
There are {{ $value }} queued up queries in query-scheduler.
|
|
|
|
expr: |
|
|
|
|
sum by (cluster, namespace) (cortex_query_scheduler_queue_length) > 1
|
|
|
|
for: 5m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
2020-07-01 09:17:43 +00:00
|
|
|
- alert: CortexCacheRequestErrors
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
Cache {{ $labels.method }} is experiencing {{ printf "%.2f" $value }}% errors.
|
|
|
|
expr: |
|
|
|
|
100 * sum by (cluster, namespace, method) (rate(cortex_cache_request_duration_seconds_count{status_code=~"5.."}[1m]))
|
|
|
|
/
|
|
|
|
sum by (cluster, namespace, method) (rate(cortex_cache_request_duration_seconds_count[1m]))
|
|
|
|
> 1
|
|
|
|
for: 15m
|
|
|
|
labels:
|
|
|
|
severity: warning
|
|
|
|
- alert: CortexIngesterRestarts
|
|
|
|
annotations:
|
2020-08-13 10:50:10 +00:00
|
|
|
message: '{{ $labels.job }}/{{ $labels.instance }} has restarted {{ printf "%.2f"
|
|
|
|
$value }} times in the last 30 mins.'
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
2021-01-20 04:06:35 +00:00
|
|
|
changes(process_start_time_seconds{job=~".+(cortex|ingester.*)"}[30m]) > 1
|
2020-07-01 09:17:43 +00:00
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexTransferFailed
|
|
|
|
annotations:
|
|
|
|
message: '{{ $labels.job }}/{{ $labels.instance }} transfer failed.'
|
|
|
|
expr: |
|
|
|
|
max_over_time(cortex_shutdown_duration_seconds_count{op="transfer",status!="success"}[15m])
|
|
|
|
for: 5m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexOldChunkInMemory
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
{{ $labels.job }}/{{ $labels.instance }} has very old unflushed chunk in memory.
|
|
|
|
expr: |
|
|
|
|
(time() - cortex_oldest_unflushed_chunk_timestamp_seconds > 36000)
|
|
|
|
and
|
|
|
|
(cortex_oldest_unflushed_chunk_timestamp_seconds > 0)
|
|
|
|
for: 5m
|
|
|
|
labels:
|
|
|
|
severity: warning
|
2020-12-15 03:27:30 +00:00
|
|
|
- alert: CortexMemoryMapAreasTooHigh
|
|
|
|
annotations:
|
|
|
|
message: '{{ $labels.job }}/{{ $labels.instance }} has a number of mmap-ed areas
|
|
|
|
close to the limit.'
|
|
|
|
expr: |
|
2021-01-20 04:06:35 +00:00
|
|
|
process_memory_map_areas{job=~".+(cortex|ingester.*|store-gateway)"} / process_memory_map_areas_limit{job=~".+(cortex|ingester.*|store-gateway)"} > 0.8
|
2020-12-15 03:27:30 +00:00
|
|
|
for: 5m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
2020-07-01 09:17:43 +00:00
|
|
|
- name: cortex_wal_alerts
|
|
|
|
rules:
|
|
|
|
- alert: CortexWALCorruption
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
{{ $labels.job }}/{{ $labels.instance }} has a corrupted WAL or checkpoint.
|
|
|
|
expr: |
|
|
|
|
increase(cortex_ingester_wal_corruptions_total[5m]) > 0
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexCheckpointCreationFailed
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
{{ $labels.job }}/{{ $labels.instance }} failed to create checkpoint.
|
|
|
|
expr: |
|
|
|
|
increase(cortex_ingester_checkpoint_creations_failed_total[10m]) > 0
|
|
|
|
labels:
|
|
|
|
severity: warning
|
|
|
|
- alert: CortexCheckpointCreationFailed
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
{{ $labels.job }}/{{ $labels.instance }} is failing to create checkpoint.
|
|
|
|
expr: |
|
|
|
|
increase(cortex_ingester_checkpoint_creations_failed_total[1h]) > 1
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexCheckpointDeletionFailed
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
{{ $labels.job }}/{{ $labels.instance }} failed to delete checkpoint.
|
|
|
|
expr: |
|
|
|
|
increase(cortex_ingester_checkpoint_deletions_failed_total[10m]) > 0
|
|
|
|
labels:
|
|
|
|
severity: warning
|
|
|
|
- alert: CortexCheckpointDeletionFailed
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
{{ $labels.instance }} is failing to delete checkpoint.
|
|
|
|
expr: |
|
|
|
|
increase(cortex_ingester_checkpoint_deletions_failed_total[2h]) > 1
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- name: cortex-provisioning
|
|
|
|
rules:
|
|
|
|
- alert: CortexProvisioningMemcachedTooSmall
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
Chunk memcached cluster is too small, should be at least {{ printf "%.2f" $value }}GB.
|
|
|
|
expr: |
|
|
|
|
(
|
|
|
|
4 *
|
|
|
|
sum by (cluster, namespace) (cortex_ingester_memory_series * cortex_ingester_chunk_size_bytes_sum / cortex_ingester_chunk_size_bytes_count)
|
|
|
|
/ 1e9
|
|
|
|
)
|
|
|
|
>
|
|
|
|
(
|
|
|
|
sum by (cluster, namespace) (memcached_limit_bytes{job=~".+/memcached"}) / 1e9
|
|
|
|
)
|
|
|
|
for: 15m
|
|
|
|
labels:
|
|
|
|
severity: warning
|
|
|
|
- alert: CortexProvisioningTooManyActiveSeries
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
Too many active series for ingesters, add more ingesters.
|
|
|
|
expr: |
|
2020-08-12 17:33:36 +00:00
|
|
|
avg by (cluster, namespace) (cortex_ingester_memory_series) > 1.6e6
|
2020-07-01 09:17:43 +00:00
|
|
|
and
|
|
|
|
sum by (cluster, namespace) (rate(cortex_ingester_received_chunks[1h])) == 0
|
|
|
|
for: 1h
|
|
|
|
labels:
|
|
|
|
severity: warning
|
|
|
|
- alert: CortexProvisioningTooManyWrites
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
High QPS for ingesters, add more ingesters.
|
|
|
|
expr: |
|
|
|
|
avg by (cluster, namespace) (rate(cortex_ingester_ingested_samples_total[1m])) > 80e3
|
|
|
|
for: 15m
|
|
|
|
labels:
|
|
|
|
severity: warning
|
2020-08-12 17:33:36 +00:00
|
|
|
- alert: CortexAllocatingTooMuchMemory
|
|
|
|
annotations:
|
|
|
|
message: |
|
2020-08-28 03:21:40 +00:00
|
|
|
Too much memory being used by {{ $labels.namespace }}/{{ $labels.pod }} - add more ingesters.
|
2020-08-12 17:33:36 +00:00
|
|
|
expr: |
|
|
|
|
(
|
2020-09-23 03:20:42 +00:00
|
|
|
container_memory_working_set_bytes{container="ingester"}
|
2020-08-12 17:33:36 +00:00
|
|
|
/
|
2020-09-23 03:20:42 +00:00
|
|
|
container_spec_memory_limit_bytes{container="ingester"}
|
2020-08-12 17:33:36 +00:00
|
|
|
) > 0.5
|
|
|
|
for: 15m
|
|
|
|
labels:
|
|
|
|
severity: warning
|
|
|
|
- alert: CortexAllocatingTooMuchMemory
|
2020-07-01 09:17:43 +00:00
|
|
|
annotations:
|
|
|
|
message: |
|
2020-08-28 03:21:40 +00:00
|
|
|
Too much memory being used by {{ $labels.namespace }}/{{ $labels.pod }} - add more ingesters.
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
2020-08-12 17:33:36 +00:00
|
|
|
(
|
2020-09-23 03:20:42 +00:00
|
|
|
container_memory_working_set_bytes{container="ingester"}
|
2020-07-01 09:17:43 +00:00
|
|
|
/
|
2020-09-23 03:20:42 +00:00
|
|
|
container_spec_memory_limit_bytes{container="ingester"}
|
2020-08-12 17:33:36 +00:00
|
|
|
) > 0.8
|
2020-07-01 09:17:43 +00:00
|
|
|
for: 15m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- name: ruler_alerts
|
|
|
|
rules:
|
|
|
|
- alert: CortexRulerFailedEvaluations
|
|
|
|
annotations:
|
|
|
|
message: |
|
2020-07-20 14:43:19 +00:00
|
|
|
Cortex Ruler {{ $labels.instance }} is experiencing {{ printf "%.2f" $value }}% errors for the rule group {{ $labels.rule_group }}.
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
2020-07-20 14:43:19 +00:00
|
|
|
sum by (cluster, namespace, instance, rule_group) (rate(cortex_prometheus_rule_evaluation_failures_total[1m]))
|
2020-07-01 09:17:43 +00:00
|
|
|
/
|
2020-07-20 14:43:19 +00:00
|
|
|
sum by (cluster, namespace, instance, rule_group) (rate(cortex_prometheus_rule_evaluations_total[1m]))
|
2020-07-01 09:17:43 +00:00
|
|
|
> 0.01
|
|
|
|
for: 5m
|
|
|
|
labels:
|
|
|
|
severity: warning
|
|
|
|
- alert: CortexRulerMissedEvaluations
|
|
|
|
annotations:
|
|
|
|
message: |
|
2020-07-20 14:43:19 +00:00
|
|
|
Cortex Ruler {{ $labels.instance }} is experiencing {{ printf "%.2f" $value }}% missed iterations for the rule group {{ $labels.rule_group }}.
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
2020-07-20 14:43:19 +00:00
|
|
|
sum by (cluster, namespace, instance, rule_group) (rate(cortex_prometheus_rule_group_iterations_missed_total[1m]))
|
2020-07-01 09:17:43 +00:00
|
|
|
/
|
2020-07-20 14:43:19 +00:00
|
|
|
sum by (cluster, namespace, instance, rule_group) (rate(cortex_prometheus_rule_group_iterations_total[1m]))
|
2020-07-01 09:17:43 +00:00
|
|
|
> 0.01
|
|
|
|
for: 5m
|
|
|
|
labels:
|
|
|
|
severity: warning
|
|
|
|
- alert: CortexRulerFailedRingCheck
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
{{ $labels.job }} is experiencing {{ printf "%.2f" $value }}% errors when checking the ring for rule group ownership.
|
|
|
|
expr: |
|
|
|
|
sum by (cluster, namespace) (rate(cortex_ruler_ring_check_errors_total[5m]))
|
|
|
|
> 0
|
|
|
|
for: 1m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- name: gossip_alerts
|
|
|
|
rules:
|
|
|
|
- alert: CortexGossipMembersMismatch
|
|
|
|
annotations:
|
2020-08-13 10:50:10 +00:00
|
|
|
message: '{{ $labels.job }}/{{ $labels.instance }} sees incorrect number of
|
|
|
|
gossip members.'
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
|
|
|
memberlist_client_cluster_members_count
|
|
|
|
!= on (cluster, namespace) group_left
|
2021-01-20 04:06:35 +00:00
|
|
|
sum by (cluster, namespace) (up{job=~".+/(distributor|ingester.*|querier|cortex|ruler)"})
|
2020-07-01 09:17:43 +00:00
|
|
|
for: 5m
|
|
|
|
labels:
|
|
|
|
severity: warning
|
|
|
|
- name: cortex_blocks_alerts
|
|
|
|
rules:
|
|
|
|
- alert: CortexIngesterHasNotShippedBlocks
|
|
|
|
annotations:
|
2020-08-13 10:50:10 +00:00
|
|
|
message: Cortex Ingester {{ $labels.namespace }}/{{ $labels.instance }} has
|
|
|
|
not shipped any block in the last 4 hours.
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
2021-01-20 04:06:35 +00:00
|
|
|
(min by(namespace, instance) (time() - thanos_objstore_bucket_last_successful_upload_time{job=~".+/ingester.*"}) > 60 * 60 * 4)
|
2020-08-12 17:33:36 +00:00
|
|
|
and
|
2021-01-20 04:06:35 +00:00
|
|
|
(max by(namespace, instance) (thanos_objstore_bucket_last_successful_upload_time{job=~".+/ingester.*"}) > 0)
|
2020-07-01 09:17:43 +00:00
|
|
|
and
|
2020-08-12 17:33:36 +00:00
|
|
|
(max by(namespace, instance) (rate(cortex_ingester_ingested_samples_total[4h])) > 0)
|
2020-07-01 09:17:43 +00:00
|
|
|
for: 15m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexIngesterHasNotShippedBlocksSinceStart
|
|
|
|
annotations:
|
2020-08-13 10:50:10 +00:00
|
|
|
message: Cortex Ingester {{ $labels.namespace }}/{{ $labels.instance }} has
|
|
|
|
not shipped any block in the last 4 hours.
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
2021-01-20 04:06:35 +00:00
|
|
|
(max by(namespace, instance) (thanos_objstore_bucket_last_successful_upload_time{job=~".+/ingester.*"}) == 0)
|
2020-08-12 17:33:36 +00:00
|
|
|
and
|
|
|
|
(max by(namespace, instance) (rate(cortex_ingester_ingested_samples_total[4h])) > 0)
|
2020-07-01 09:17:43 +00:00
|
|
|
for: 4h
|
|
|
|
labels:
|
|
|
|
severity: critical
|
2021-01-27 03:31:43 +00:00
|
|
|
- alert: CortexIngesterHasUnshippedBlocks
|
|
|
|
annotations:
|
|
|
|
message: Cortex Ingester {{ $labels.namespace }}/{{ $labels.instance }} has
|
|
|
|
compacted a block {{ $value | humanizeDuration }} ago but it hasn't been successfully
|
|
|
|
uploaded to the storage yet.
|
|
|
|
expr: |
|
|
|
|
(time() - cortex_ingester_oldest_unshipped_block_timestamp_seconds > 3600)
|
|
|
|
and
|
|
|
|
(cortex_ingester_oldest_unshipped_block_timestamp_seconds > 0)
|
|
|
|
for: 15m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
2020-08-12 17:33:36 +00:00
|
|
|
- alert: CortexIngesterTSDBHeadCompactionFailed
|
|
|
|
annotations:
|
2020-08-13 10:50:10 +00:00
|
|
|
message: Cortex Ingester {{ $labels.namespace }}/{{ $labels.instance }} is failing
|
|
|
|
to compact TSDB head.
|
2020-08-12 17:33:36 +00:00
|
|
|
expr: |
|
|
|
|
rate(cortex_ingester_tsdb_compactions_failed_total[5m]) > 0
|
|
|
|
for: 15m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
2020-11-11 03:16:57 +00:00
|
|
|
- alert: CortexIngesterTSDBHeadTruncationFailed
|
|
|
|
annotations:
|
|
|
|
message: Cortex Ingester {{ $labels.namespace }}/{{ $labels.instance }} is failing
|
|
|
|
to truncate TSDB head.
|
|
|
|
expr: |
|
|
|
|
rate(cortex_ingester_tsdb_head_truncations_failed_total[5m]) > 0
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexIngesterTSDBCheckpointCreationFailed
|
|
|
|
annotations:
|
|
|
|
message: Cortex Ingester {{ $labels.namespace }}/{{ $labels.instance }} is failing
|
|
|
|
to create TSDB checkpoint.
|
|
|
|
expr: |
|
|
|
|
rate(cortex_ingester_tsdb_checkpoint_creations_failed_total[5m]) > 0
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexIngesterTSDBCheckpointDeletionFailed
|
|
|
|
annotations:
|
|
|
|
message: Cortex Ingester {{ $labels.namespace }}/{{ $labels.instance }} is failing
|
|
|
|
to delete TSDB checkpoint.
|
|
|
|
expr: |
|
|
|
|
rate(cortex_ingester_tsdb_checkpoint_deletions_failed_total[5m]) > 0
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexIngesterTSDBWALTruncationFailed
|
|
|
|
annotations:
|
|
|
|
message: Cortex Ingester {{ $labels.namespace }}/{{ $labels.instance }} is failing
|
|
|
|
to truncate TSDB WAL.
|
|
|
|
expr: |
|
|
|
|
rate(cortex_ingester_tsdb_wal_truncations_failed_total[5m]) > 0
|
|
|
|
labels:
|
|
|
|
severity: warning
|
|
|
|
- alert: CortexIngesterTSDBWALCorrupted
|
|
|
|
annotations:
|
|
|
|
message: Cortex Ingester {{ $labels.namespace }}/{{ $labels.instance }} got
|
|
|
|
a corrupted TSDB WAL.
|
|
|
|
expr: |
|
|
|
|
rate(cortex_ingester_tsdb_wal_corruptions_total[5m]) > 0
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexIngesterTSDBWALWritesFailed
|
|
|
|
annotations:
|
|
|
|
message: Cortex Ingester {{ $labels.namespace }}/{{ $labels.instance }} is failing
|
|
|
|
to write to TSDB WAL.
|
|
|
|
expr: |
|
|
|
|
rate(cortex_ingester_tsdb_wal_writes_failed_total[1m]) > 0
|
|
|
|
for: 3m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
2020-07-01 09:17:43 +00:00
|
|
|
- alert: CortexQuerierHasNotScanTheBucket
|
|
|
|
annotations:
|
2020-08-13 10:50:10 +00:00
|
|
|
message: Cortex Querier {{ $labels.namespace }}/{{ $labels.instance }} has not
|
|
|
|
successfully scanned the bucket since {{ $value | humanizeDuration }}.
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
|
|
|
(time() - cortex_querier_blocks_last_successful_scan_timestamp_seconds > 60 * 30)
|
|
|
|
and
|
|
|
|
cortex_querier_blocks_last_successful_scan_timestamp_seconds > 0
|
|
|
|
for: 5m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexQuerierHighRefetchRate
|
|
|
|
annotations:
|
2020-08-13 10:50:10 +00:00
|
|
|
message: Cortex Queries in {{ $labels.namespace }} are refetching series from
|
|
|
|
different store-gateways (because of missing blocks) for the {{ printf "%.0f"
|
|
|
|
$value }}% of queries.
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
|
|
|
100 * (
|
|
|
|
(
|
|
|
|
sum by(namespace) (rate(cortex_querier_storegateway_refetches_per_query_count[5m]))
|
|
|
|
-
|
|
|
|
sum by(namespace) (rate(cortex_querier_storegateway_refetches_per_query_bucket{le="0"}[5m]))
|
|
|
|
)
|
|
|
|
/
|
|
|
|
sum by(namespace) (rate(cortex_querier_storegateway_refetches_per_query_count[5m]))
|
|
|
|
)
|
|
|
|
> 1
|
|
|
|
for: 10m
|
|
|
|
labels:
|
|
|
|
severity: warning
|
|
|
|
- alert: CortexStoreGatewayHasNotSyncTheBucket
|
|
|
|
annotations:
|
2020-08-13 10:50:10 +00:00
|
|
|
message: Cortex Store Gateway {{ $labels.namespace }}/{{ $labels.instance }}
|
|
|
|
has not successfully synched the bucket since {{ $value | humanizeDuration
|
|
|
|
}}.
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
|
|
|
(time() - cortex_bucket_stores_blocks_last_successful_sync_timestamp_seconds{component="store-gateway"} > 60 * 30)
|
|
|
|
and
|
|
|
|
cortex_bucket_stores_blocks_last_successful_sync_timestamp_seconds{component="store-gateway"} > 0
|
|
|
|
for: 5m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
2021-01-06 03:45:40 +00:00
|
|
|
- alert: CortexBucketIndexNotUpdated
|
|
|
|
annotations:
|
|
|
|
message: Cortex bucket index for tenant {{ $labels.user }} in {{ $labels.namespace
|
|
|
|
}} has not been updated since {{ $value | humanizeDuration }}.
|
|
|
|
expr: |
|
|
|
|
min by(namespace, user) (time() - cortex_bucket_index_last_successful_update_timestamp_seconds) > 7200
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexTenantHasPartialBlocks
|
|
|
|
annotations:
|
|
|
|
message: Cortex tenant {{ $labels.user }} in {{ $labels.namespace }} has {{
|
|
|
|
$value }} partial blocks.
|
|
|
|
expr: |
|
|
|
|
max by(namespace, user) (cortex_bucket_blocks_partials_count) > 0
|
|
|
|
for: 6h
|
|
|
|
labels:
|
|
|
|
severity: warning
|
2020-07-01 09:17:43 +00:00
|
|
|
- name: cortex_compactor_alerts
|
|
|
|
rules:
|
|
|
|
- alert: CortexCompactorHasNotSuccessfullyCleanedUpBlocks
|
|
|
|
annotations:
|
2020-08-13 10:50:10 +00:00
|
|
|
message: Cortex Compactor {{ $labels.namespace }}/{{ $labels.instance }} has
|
|
|
|
not successfully cleaned up blocks in the last 24 hours.
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
|
|
|
(time() - cortex_compactor_block_cleanup_last_successful_run_timestamp_seconds > 60 * 60 * 24)
|
|
|
|
and
|
|
|
|
(cortex_compactor_block_cleanup_last_successful_run_timestamp_seconds > 0)
|
|
|
|
for: 15m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexCompactorHasNotSuccessfullyCleanedUpBlocksSinceStart
|
|
|
|
annotations:
|
2020-08-13 10:50:10 +00:00
|
|
|
message: Cortex Compactor {{ $labels.namespace }}/{{ $labels.instance }} has
|
|
|
|
not successfully cleaned up blocks in the last 24 hours.
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
|
|
|
cortex_compactor_block_cleanup_last_successful_run_timestamp_seconds == 0
|
|
|
|
for: 24h
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexCompactorHasNotUploadedBlocks
|
|
|
|
annotations:
|
2020-08-13 10:50:10 +00:00
|
|
|
message: Cortex Compactor {{ $labels.namespace }}/{{ $labels.instance }} has
|
|
|
|
not uploaded any block in the last 24 hours.
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
|
|
|
(time() - thanos_objstore_bucket_last_successful_upload_time{job=~".+/compactor"} > 60 * 60 * 24)
|
|
|
|
and
|
|
|
|
(thanos_objstore_bucket_last_successful_upload_time{job=~".+/compactor"} > 0)
|
|
|
|
for: 15m
|
|
|
|
labels:
|
|
|
|
severity: critical
|
|
|
|
- alert: CortexCompactorHasNotUploadedBlocksSinceStart
|
|
|
|
annotations:
|
2020-08-13 10:50:10 +00:00
|
|
|
message: Cortex Compactor {{ $labels.namespace }}/{{ $labels.instance }} has
|
|
|
|
not uploaded any block in the last 24 hours.
|
2020-07-01 09:17:43 +00:00
|
|
|
expr: |
|
|
|
|
thanos_objstore_bucket_last_successful_upload_time{job=~".+/compactor"} == 0
|
|
|
|
for: 24h
|
|
|
|
labels:
|
|
|
|
severity: critical
|
2020-11-11 03:16:57 +00:00
|
|
|
- alert: CortexCompactorRunFailed
|
|
|
|
annotations:
|
|
|
|
message: |
|
|
|
|
{{ $labels.job }}/{{ $labels.instance }} failed to run compaction.
|
|
|
|
expr: |
|
|
|
|
increase(cortex_compactor_runs_failed_total[2h]) > 1
|
|
|
|
labels:
|
|
|
|
severity: critical
|