2020-07-01 09:17:43 +00:00
|
|
|
groups:
|
|
|
|
|
- name: cortex_api
|
|
|
|
|
rules:
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job) / sum(rate(cortex_request_duration_seconds_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_request_duration_seconds:avg
|
|
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job)
|
|
|
|
|
record: cluster_job:cortex_request_duration_seconds_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_request_duration_seconds_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, route))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_route:cortex_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, route))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_route:cortex_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)
|
|
|
|
|
/ sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_route:cortex_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job,
|
|
|
|
|
route)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)
|
|
|
|
|
record: cluster_job_route:cortex_request_duration_seconds_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
|
|
|
|
|
record: cluster_job_route:cortex_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, namespace, job, route))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_namespace_job_route:cortex_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, namespace, job, route))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_namespace_job_route:cortex_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
|
|
|
|
|
job, route) / sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
namespace, job, route)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_namespace_job_route:cortex_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, namespace,
|
|
|
|
|
job, route)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_namespace_job_route:cortex_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
|
|
|
|
|
job, route)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_namespace_job_route:cortex_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, namespace,
|
|
|
|
|
job, route)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_namespace_job_route:cortex_request_duration_seconds_count:sum_rate
|
|
|
|
|
- name: cortex_cache
|
|
|
|
|
rules:
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_memcache_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, method))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_memcache_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_memcache_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, method))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_memcache_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_memcache_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, method) / sum(rate(cortex_memcache_request_duration_seconds_count[1m]))
|
|
|
|
|
by (cluster, job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_memcache_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_memcache_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_memcache_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_memcache_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_memcache_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_memcache_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_memcache_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_cache_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_cache_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job)
|
|
|
|
|
/ sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_cache_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_cache_request_duration_seconds_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_cache_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_cache_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, method))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_cache_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, method))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_cache_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job,
|
|
|
|
|
method) / sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_cache_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_cache_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job,
|
|
|
|
|
method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_cache_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_cache_request_duration_seconds_count:sum_rate
|
|
|
|
|
- name: cortex_storage
|
|
|
|
|
rules:
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_bigtable_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_bigtable_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_bigtable_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_bigtable_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_bigtable_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, operation) / sum(rate(cortex_bigtable_request_duration_seconds_count[1m]))
|
|
|
|
|
by (cluster, job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_bigtable_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_bigtable_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_bigtable_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_bigtable_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_bigtable_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_bigtable_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_bigtable_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_cassandra_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_cassandra_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_cassandra_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_cassandra_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cassandra_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, operation) / sum(rate(cortex_cassandra_request_duration_seconds_count[1m]))
|
|
|
|
|
by (cluster, job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_cassandra_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cassandra_request_duration_seconds_bucket[1m])) by (le,
|
|
|
|
|
cluster, job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_cassandra_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cassandra_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_cassandra_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cassandra_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_cassandra_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_dynamo_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_dynamo_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_dynamo_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_dynamo_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_dynamo_request_duration_seconds_sum[1m])) by (cluster, job,
|
|
|
|
|
operation) / sum(rate(cortex_dynamo_request_duration_seconds_count[1m])) by
|
|
|
|
|
(cluster, job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_dynamo_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_dynamo_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_dynamo_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_dynamo_request_duration_seconds_sum[1m])) by (cluster, job,
|
|
|
|
|
operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_dynamo_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_dynamo_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_dynamo_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_chunk_store_index_lookups_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_index_lookups_per_query:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_chunk_store_index_lookups_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_index_lookups_per_query:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_index_lookups_per_query_sum[1m])) by (cluster,
|
|
|
|
|
job) / sum(rate(cortex_chunk_store_index_lookups_per_query_count[1m])) by (cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_index_lookups_per_query:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_index_lookups_per_query_bucket[1m])) by (le,
|
|
|
|
|
cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_index_lookups_per_query_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_index_lookups_per_query_sum[1m])) by (cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_index_lookups_per_query_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_index_lookups_per_query_count[1m])) by (cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_index_lookups_per_query_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_chunk_store_series_pre_intersection_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_pre_intersection_per_query:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_chunk_store_series_pre_intersection_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_pre_intersection_per_query:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_pre_intersection_per_query_sum[1m]))
|
|
|
|
|
by (cluster, job) / sum(rate(cortex_chunk_store_series_pre_intersection_per_query_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_pre_intersection_per_query:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_pre_intersection_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_pre_intersection_per_query_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_pre_intersection_per_query_sum[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_pre_intersection_per_query_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_pre_intersection_per_query_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_pre_intersection_per_query_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_chunk_store_series_post_intersection_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_post_intersection_per_query:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_chunk_store_series_post_intersection_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_post_intersection_per_query:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_post_intersection_per_query_sum[1m]))
|
|
|
|
|
by (cluster, job) / sum(rate(cortex_chunk_store_series_post_intersection_per_query_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_post_intersection_per_query:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_post_intersection_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_post_intersection_per_query_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_post_intersection_per_query_sum[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_post_intersection_per_query_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_post_intersection_per_query_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_post_intersection_per_query_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_chunk_store_chunks_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_chunks_per_query:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_chunk_store_chunks_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_chunks_per_query:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_chunks_per_query_sum[1m])) by (cluster, job)
|
|
|
|
|
/ sum(rate(cortex_chunk_store_chunks_per_query_count[1m])) by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_chunks_per_query:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_chunks_per_query_bucket[1m])) by (le, cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_chunks_per_query_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_chunk_store_chunks_per_query_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_chunk_store_chunks_per_query_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_chunk_store_chunks_per_query_count[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_chunk_store_chunks_per_query_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_database_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, method))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_database_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_database_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, method))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_database_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_database_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, method) / sum(rate(cortex_database_request_duration_seconds_count[1m]))
|
|
|
|
|
by (cluster, job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_database_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_database_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_database_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_database_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_database_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_database_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_database_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_gcs_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_gcs_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_gcs_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_gcs_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_gcs_request_duration_seconds_sum[1m])) by (cluster, job,
|
|
|
|
|
operation) / sum(rate(cortex_gcs_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_gcs_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_gcs_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_gcs_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_gcs_request_duration_seconds_sum[1m])) by (cluster, job,
|
|
|
|
|
operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_gcs_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_gcs_request_duration_seconds_count[1m])) by (cluster, job,
|
|
|
|
|
operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_gcs_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_kv_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_kv_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)
|
|
|
|
|
/ sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_kv_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_kv_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_kv_request_duration_seconds_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_kv_request_duration_seconds_count:sum_rate
|
|
|
|
|
- name: cortex_queries
|
|
|
|
|
rules:
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_retries:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_retries:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job) / sum(rate(cortex_query_frontend_retries_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_retries:avg
|
|
|
|
|
- expr: sum(rate(cortex_query_frontend_retries_bucket[1m])) by (le, cluster, job)
|
|
|
|
|
record: cluster_job:cortex_query_frontend_retries_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_query_frontend_retries_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_query_frontend_retries_count[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_query_frontend_retries_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_queue_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_queue_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job) / sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by
|
|
|
|
|
(cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_queue_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m])) by (le,
|
|
|
|
|
cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_queue_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_queue_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_queue_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_series_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_series:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_series_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_series:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_series_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_series:avg
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_series_bucket[1m])) by (le, cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_series_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_series_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_series_count[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_series_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_chunks_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_chunks:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_chunks_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_chunks:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_ingester_queried_chunks_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_chunks_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_chunks:avg
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_chunks_bucket[1m])) by (le, cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_chunks_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_chunks_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_chunks_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_chunks_count[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_chunks_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_samples:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_samples:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_samples_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_samples:avg
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_samples_bucket[1m])) by (le, cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_samples_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_samples_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_samples_count[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_samples_count:sum_rate
|
|
|
|
|
- name: cortex_received_samples
|
|
|
|
|
rules:
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, namespace, job) (rate(cortex_distributor_received_samples_total[5m]))
|
|
|
|
|
record: cluster_namespace_job:cortex_distributor_received_samples:rate5m
|
