2020-07-01 09:17:43 +00:00
|
|
|
groups:
|
2021-10-05 03:23:23 +00:00
|
|
|
- name: cortex_api_1
|
2020-07-01 09:17:43 +00:00
|
|
|
rules:
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job) / sum(rate(cortex_request_duration_seconds_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_request_duration_seconds:avg
|
|
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job)
|
|
|
|
|
record: cluster_job:cortex_request_duration_seconds_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_request_duration_seconds_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_request_duration_seconds_count:sum_rate
|
2021-10-05 03:23:23 +00:00
|
|
|
- name: cortex_api_2
|
|
|
|
|
rules:
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, route))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_route:cortex_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, route))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_route:cortex_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)
|
|
|
|
|
/ sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_route:cortex_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, job,
|
|
|
|
|
route)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, job, route)
|
|
|
|
|
record: cluster_job_route:cortex_request_duration_seconds_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, job, route)
|
|
|
|
|
record: cluster_job_route:cortex_request_duration_seconds_count:sum_rate
|
2021-10-05 03:23:23 +00:00
|
|
|
- name: cortex_api_3
|
|
|
|
|
rules:
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, namespace, job, route))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_namespace_job_route:cortex_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, namespace, job, route))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_namespace_job_route:cortex_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
|
|
|
|
|
job, route) / sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
namespace, job, route)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_namespace_job_route:cortex_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_bucket[1m])) by (le, cluster, namespace,
|
|
|
|
|
job, route)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_namespace_job_route:cortex_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_sum[1m])) by (cluster, namespace,
|
|
|
|
|
job, route)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_namespace_job_route:cortex_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_request_duration_seconds_count[1m])) by (cluster, namespace,
|
|
|
|
|
job, route)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_namespace_job_route:cortex_request_duration_seconds_count:sum_rate
|
2020-10-14 03:23:36 +00:00
|
|
|
- name: cortex_querier_api
|
|
|
|
|
rules:
|
|
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
|
|
|
|
record: cluster_job:cortex_querier_request_duration_seconds:99quantile
|
|
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
|
|
|
|
record: cluster_job:cortex_querier_request_duration_seconds:50quantile
|
|
|
|
|
- expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job) / sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job)
|
|
|
|
|
record: cluster_job:cortex_querier_request_duration_seconds:avg
|
|
|
|
|
- expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job)
|
|
|
|
|
record: cluster_job:cortex_querier_request_duration_seconds_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job)
|
|
|
|
|
record: cluster_job:cortex_querier_request_duration_seconds_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job)
|
|
|
|
|
record: cluster_job:cortex_querier_request_duration_seconds_count:sum_rate
|
|
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, route))
|
|
|
|
|
record: cluster_job_route:cortex_querier_request_duration_seconds:99quantile
|
|
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, route))
|
|
|
|
|
record: cluster_job_route:cortex_querier_request_duration_seconds:50quantile
|
|
|
|
|
- expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, route) / sum(rate(cortex_querier_request_duration_seconds_count[1m])) by
|
|
|
|
|
(cluster, job, route)
|
|
|
|
|
record: cluster_job_route:cortex_querier_request_duration_seconds:avg
|
|
|
|
|
- expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job, route)
|
|
|
|
|
record: cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, route)
|
|
|
|
|
record: cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, route)
|
|
|
|
|
record: cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate
|
|
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, namespace, job, route))
|
|
|
|
|
record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:99quantile
|
|
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_querier_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, namespace, job, route))
|
|
|
|
|
record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:50quantile
|
|
|
|
|
- expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
namespace, job, route) / sum(rate(cortex_querier_request_duration_seconds_count[1m]))
|
|
|
|
|
by (cluster, namespace, job, route)
|
|
|
|
|
record: cluster_namespace_job_route:cortex_querier_request_duration_seconds:avg
|
|
|
|
|
- expr: sum(rate(cortex_querier_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
namespace, job, route)
|
|
|
|
|
record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_querier_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
namespace, job, route)
|
|
|
|
|
record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_querier_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
namespace, job, route)
|
|
|
|
|
record: cluster_namespace_job_route:cortex_querier_request_duration_seconds_count:sum_rate
|
2020-07-01 09:17:43 +00:00
|
|
|
- name: cortex_cache
|
|
|
|
|
rules:
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_memcache_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, method))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_memcache_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_memcache_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, method))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_memcache_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_memcache_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, method) / sum(rate(cortex_memcache_request_duration_seconds_count[1m]))
|
|
|
|
|
by (cluster, job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_memcache_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_memcache_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_memcache_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_memcache_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_memcache_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_memcache_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_memcache_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_cache_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_cache_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job)
|
|
|
|
|
/ sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_cache_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_cache_request_duration_seconds_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_cache_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_cache_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, method))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_cache_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, method))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_cache_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job,
|
|
|
|
|
method) / sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_cache_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_cache_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_sum[1m])) by (cluster, job,
|
|
|
|
|
method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_cache_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cache_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_cache_request_duration_seconds_count:sum_rate
|
|
|
|
|
- name: cortex_storage
|
|
|
|
|
rules:
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_bigtable_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_bigtable_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_bigtable_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_bigtable_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_bigtable_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, operation) / sum(rate(cortex_bigtable_request_duration_seconds_count[1m]))
|
|
|
|
|
by (cluster, job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_bigtable_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_bigtable_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_bigtable_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_bigtable_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_bigtable_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_bigtable_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_bigtable_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_cassandra_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_cassandra_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_cassandra_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_cassandra_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cassandra_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, operation) / sum(rate(cortex_cassandra_request_duration_seconds_count[1m]))
|
|
|
|
|
by (cluster, job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_cassandra_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cassandra_request_duration_seconds_bucket[1m])) by (le,
|
|
|
|
|
cluster, job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_cassandra_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cassandra_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_cassandra_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_cassandra_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_cassandra_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_dynamo_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_dynamo_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_dynamo_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_dynamo_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_dynamo_request_duration_seconds_sum[1m])) by (cluster, job,
|
|
|
|
|
operation) / sum(rate(cortex_dynamo_request_duration_seconds_count[1m])) by
|
|
|
|
|
(cluster, job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_dynamo_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_dynamo_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_dynamo_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_dynamo_request_duration_seconds_sum[1m])) by (cluster, job,
|
|
|
|
|
operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_dynamo_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_dynamo_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_dynamo_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_chunk_store_index_lookups_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_index_lookups_per_query:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_chunk_store_index_lookups_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_index_lookups_per_query:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_index_lookups_per_query_sum[1m])) by (cluster,
|
|
|
|
|
job) / sum(rate(cortex_chunk_store_index_lookups_per_query_count[1m])) by (cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_index_lookups_per_query:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_index_lookups_per_query_bucket[1m])) by (le,
|
|
|
|
|
cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_index_lookups_per_query_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_index_lookups_per_query_sum[1m])) by (cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_index_lookups_per_query_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_index_lookups_per_query_count[1m])) by (cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_index_lookups_per_query_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_chunk_store_series_pre_intersection_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_pre_intersection_per_query:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_chunk_store_series_pre_intersection_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_pre_intersection_per_query:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_pre_intersection_per_query_sum[1m]))
|
|
|
|
|
by (cluster, job) / sum(rate(cortex_chunk_store_series_pre_intersection_per_query_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_pre_intersection_per_query:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_pre_intersection_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_pre_intersection_per_query_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_pre_intersection_per_query_sum[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_pre_intersection_per_query_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_pre_intersection_per_query_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_pre_intersection_per_query_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_chunk_store_series_post_intersection_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_post_intersection_per_query:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_chunk_store_series_post_intersection_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_post_intersection_per_query:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_post_intersection_per_query_sum[1m]))
|
|
|
|
|
by (cluster, job) / sum(rate(cortex_chunk_store_series_post_intersection_per_query_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_post_intersection_per_query:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_post_intersection_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_post_intersection_per_query_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_post_intersection_per_query_sum[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_post_intersection_per_query_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_series_post_intersection_per_query_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_series_post_intersection_per_query_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_chunk_store_chunks_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_chunks_per_query:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_chunk_store_chunks_per_query_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_chunks_per_query:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_chunks_per_query_sum[1m])) by (cluster, job)
|
|
|
|
|
/ sum(rate(cortex_chunk_store_chunks_per_query_count[1m])) by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_chunks_per_query:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_chunk_store_chunks_per_query_bucket[1m])) by (le, cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_chunk_store_chunks_per_query_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_chunk_store_chunks_per_query_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_chunk_store_chunks_per_query_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_chunk_store_chunks_per_query_count[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_chunk_store_chunks_per_query_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_database_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, method))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_database_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_database_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, method))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_database_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_database_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, method) / sum(rate(cortex_database_request_duration_seconds_count[1m]))
|
|
|
|
|
by (cluster, job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_database_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_database_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_database_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_database_request_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_database_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_database_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, method)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_method:cortex_database_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_gcs_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_gcs_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_gcs_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job, operation))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_gcs_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_gcs_request_duration_seconds_sum[1m])) by (cluster, job,
|
|
|
|
|
operation) / sum(rate(cortex_gcs_request_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_gcs_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_gcs_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job, operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_gcs_request_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_gcs_request_duration_seconds_sum[1m])) by (cluster, job,
|
|
|
|
|
operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_gcs_request_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_gcs_request_duration_seconds_count[1m])) by (cluster, job,
|
|
|
|
|
operation)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job_operation:cortex_gcs_request_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_kv_request_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_kv_request_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)
|
|
|
|
|
/ sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_kv_request_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_kv_request_duration_seconds_bucket[1m])) by (le, cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_kv_request_duration_seconds_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_kv_request_duration_seconds_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_kv_request_duration_seconds_count[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_kv_request_duration_seconds_count:sum_rate
|
|
|
|
|
- name: cortex_queries
|
|
|
|
|
rules:
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_retries:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_retries:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job) / sum(rate(cortex_query_frontend_retries_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_retries:avg
|
|
|
|
|
- expr: sum(rate(cortex_query_frontend_retries_bucket[1m])) by (le, cluster, job)
|
|
|
|
|
record: cluster_job:cortex_query_frontend_retries_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_query_frontend_retries_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_query_frontend_retries_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_query_frontend_retries_count[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_query_frontend_retries_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_queue_duration_seconds:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_queue_duration_seconds:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job) / sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by
|
|
|
|
|
(cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_queue_duration_seconds:avg
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_query_frontend_queue_duration_seconds_bucket[1m])) by (le,
|
|
|
|
|
cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_queue_duration_seconds_bucket:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_query_frontend_queue_duration_seconds_sum[1m])) by (cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_queue_duration_seconds_sum:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by (cluster,
|
|
|
|
|
job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_query_frontend_queue_duration_seconds_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_series_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_series:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_series_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_series:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_series_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_series:avg
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_series_bucket[1m])) by (le, cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_series_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_series_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_series_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_series_count[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_series_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_chunks_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_chunks:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_chunks_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_chunks:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_ingester_queried_chunks_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_chunks_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_chunks:avg
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_chunks_bucket[1m])) by (le, cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_chunks_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_chunks_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_chunks_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_chunks_count[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_chunks_count:sum_rate
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_samples:99quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: histogram_quantile(0.50, sum(rate(cortex_ingester_queried_samples_bucket[1m]))
|
|
|
|
|
by (le, cluster, job))
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_samples:50quantile
|
2020-08-13 10:50:10 +00:00
|
|
|
- expr: sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job) / sum(rate(cortex_ingester_queried_samples_count[1m]))
|
|
|
|
|
by (cluster, job)
|
2020-07-01 09:17:43 +00:00
|
|
|
record: cluster_job:cortex_ingester_queried_samples:avg
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_samples_bucket[1m])) by (le, cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_samples_bucket:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_samples_sum[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_samples_sum:sum_rate
|
|
|
|
|
- expr: sum(rate(cortex_ingester_queried_samples_count[1m])) by (cluster, job)
|
|
|
|
|
record: cluster_job:cortex_ingester_queried_samples_count:sum_rate
|
|
|
|
|
- name: cortex_received_samples
|
|
|
|
|
rules:
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, namespace, job) (rate(cortex_distributor_received_samples_total[5m]))
|
|
|
|
|
record: cluster_namespace_job:cortex_distributor_received_samples:rate5m
|
2021-03-20 03:27:27 +00:00
|
|
|
- name: cortex_scaling_rules
|
|
|
|
|
rules:
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, namespace, deployment) (
|
2021-07-29 03:25:46 +00:00
|
|
|
label_replace(
|
|
|
|
|
kube_deployment_spec_replicas,
|
|
|
|
|
# The question mark in "(.*?)" is used to make it non-greedy, otherwise it
|
|
|
|
|
# always matches everything and the (optional) zone is not removed.
|
|
|
|
|
"deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
or
|
|
|
|
|
sum by (cluster, namespace, deployment) (
|
|
|
|
|
label_replace(kube_statefulset_replicas, "deployment", "$1", "statefulset", "(.*?)(?:-zone-[a-z])?")
|
2021-03-20 03:27:27 +00:00
|
|
|
)
|
|
|
|
|
record: cluster_namespace_deployment:actual_replicas:count
|
|
|
|
|
- expr: |
|
|
|
|
|
ceil(
|
|
|
|
|
quantile_over_time(0.99,
|
|
|
|
|
sum by (cluster, namespace) (
|
|
|
|
|
cluster_namespace_job:cortex_distributor_received_samples:rate5m
|
|
|
|
|
)[24h:]
|
|
|
|
|
)
|
|
|
|
|
/ 240000
|
|
|
|
|
)
|
|
|
|
|
labels:
|
|
|
|
|
deployment: distributor
|
|
|
|
|
reason: sample_rate
|
|
|
|
|
record: cluster_namespace_deployment_reason:required_replicas:count
|
|
|
|
|
- expr: |
|
|
|
|
|
ceil(
|
|
|
|
|
sum by (cluster, namespace) (cortex_overrides{limit_name="ingestion_rate"})
|
|
|
|
|
* 0.59999999999999998 / 240000
|
|
|
|
|
)
|
|
|
|
|
labels:
|
|
|
|
|
deployment: distributor
|
|
|
|
|
reason: sample_rate_limits
|
|
|
|
|
record: cluster_namespace_deployment_reason:required_replicas:count
|
|
|
|
|
- expr: |
|
|
|
|
|
ceil(
|
|
|
|
|
quantile_over_time(0.99,
|
|
|
|
|
sum by (cluster, namespace) (
|
|
|
|
|
cluster_namespace_job:cortex_distributor_received_samples:rate5m
|
|
|
|
|
)[24h:]
|
|
|
|
|
)
|
|
|
|
|
* 3 / 80000
|
|
|
|
|
)
|
|
|
|
|
labels:
|
|
|
|
|
deployment: ingester
|
|
|
|
|
reason: sample_rate
|
|
|
|
|
record: cluster_namespace_deployment_reason:required_replicas:count
|
|
|
|
|
- expr: |
|
|
|
|
|
ceil(
|
|
|
|
|
quantile_over_time(0.99,
|
|
|
|
|
sum by(cluster, namespace) (
|
|
|
|
|
cortex_ingester_memory_series
|
|
|
|
|
)[24h:]
|
|
|
|
|
)
|
2024-10-25 03:33:09 +00:00
|
|
|
/ 3000000
|
2021-03-20 03:27:27 +00:00
|
|
|
)
|
|
|
|
|
labels:
|
|
|
|
|
deployment: ingester
|
|
|
|
|
reason: active_series
|
|
|
|
|
record: cluster_namespace_deployment_reason:required_replicas:count
|
|
|
|
|
- expr: |
|
|
|
|
|
ceil(
|
|
|
|
|
sum by (cluster, namespace) (cortex_overrides{limit_name="max_global_series_per_user"})
|
2024-10-25 03:33:09 +00:00
|
|
|
* 3 * 0.59999999999999998 / 3000000
|
2021-03-20 03:27:27 +00:00
|
|
|
)
|
|
|
|
|
labels:
|
|
|
|
|
deployment: ingester
|
|
|
|
|
reason: active_series_limits
|
|
|
|
|
record: cluster_namespace_deployment_reason:required_replicas:count
|
|
|
|
|
- expr: |
|
|
|
|
|
ceil(
|
|
|
|
|
sum by (cluster, namespace) (cortex_overrides{limit_name="ingestion_rate"})
|
|
|
|
|
* 0.59999999999999998 / 80000
|
|
|
|
|
)
|
|
|
|
|
labels:
|
|
|
|
|
deployment: ingester
|
|
|
|
|
reason: sample_rate_limits
|
|
|
|
|
record: cluster_namespace_deployment_reason:required_replicas:count
|
|
|
|
|
- expr: |
|
|
|
|
|
ceil(
|
|
|
|
|
(sum by (cluster, namespace) (
|
2021-07-29 03:25:46 +00:00
|
|
|
cortex_ingester_tsdb_storage_blocks_bytes{job=~".+/ingester.*"}
|
2021-03-20 03:27:27 +00:00
|
|
|
) / 4)
|
|
|
|
|
/
|
|
|
|
|
avg by (cluster, namespace) (
|
|
|
|
|
memcached_limit_bytes{job=~".+/memcached"}
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
labels:
|
|
|
|
|
deployment: memcached
|
|
|
|
|
reason: active_series
|
|
|
|
|
record: cluster_namespace_deployment_reason:required_replicas:count
|
2021-04-01 03:34:23 +00:00
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, namespace, deployment) (
|
|
|
|
|
label_replace(
|
2021-07-29 03:25:46 +00:00
|
|
|
label_replace(
|
2021-08-26 03:23:05 +00:00
|
|
|
node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate,
|
2021-07-29 03:25:46 +00:00
|
|
|
"deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
|
|
|
|
|
),
|
|
|
|
|
# The question mark in "(.*?)" is used to make it non-greedy, otherwise it
|
|
|
|
|
# always matches everything and the (optional) zone is not removed.
|
|
|
|
|
"deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
|
2021-04-01 03:34:23 +00:00
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
record: cluster_namespace_deployment:container_cpu_usage_seconds_total:sum_rate
|
|
|
|
|
- expr: |
|
2021-06-02 05:48:33 +00:00
|
|
|
# This recording rule is made compatible with the breaking changes introduced in kube-state-metrics v2
|
|
|
|
|
# that remove resource metrics, ref:
|
|
|
|
|
# - https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
|
|
|
|
|
# - https://github.com/kubernetes/kube-state-metrics/pull/1004
|
|
|
|
|
#
|
|
|
|
|
# This is the old expression, compatible with kube-state-metrics < v2.0.0,
|
|
|
|
|
# where kube_pod_container_resource_requests_cpu_cores was removed:
|
|
|
|
|
(
|
|
|
|
|
sum by (cluster, namespace, deployment) (
|
|
|
|
|
label_replace(
|
2021-07-29 03:25:46 +00:00
|
|
|
label_replace(
|
|
|
|
|
kube_pod_container_resource_requests_cpu_cores,
|
|
|
|
|
"deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
|
|
|
|
|
),
|
|
|
|
|
# The question mark in "(.*?)" is used to make it non-greedy, otherwise it
|
|
|
|
|
# always matches everything and the (optional) zone is not removed.
|
|
|
|
|
"deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
|
2021-06-02 05:48:33 +00:00
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
or
|
|
|
|
|
# This expression is compatible with kube-state-metrics >= v1.4.0,
|
|
|
|
|
# where kube_pod_container_resource_requests was introduced.
|
|
|
|
|
(
|
|
|
|
|
sum by (cluster, namespace, deployment) (
|
|
|
|
|
label_replace(
|
2021-07-29 03:25:46 +00:00
|
|
|
label_replace(
|
|
|
|
|
kube_pod_container_resource_requests{resource="cpu"},
|
|
|
|
|
"deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
|
|
|
|
|
),
|
|
|
|
|
# The question mark in "(.*?)" is used to make it non-greedy, otherwise it
|
|
|
|
|
# always matches everything and the (optional) zone is not removed.
|
|
|
|
|
"deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
|
2021-06-02 05:48:33 +00:00
|
|
|
)
|
2021-04-01 03:34:23 +00:00
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
record: cluster_namespace_deployment:kube_pod_container_resource_requests_cpu_cores:sum
|
2021-03-20 03:27:27 +00:00
|
|
|
- expr: |
|
|
|
|
|
ceil(
|
|
|
|
|
cluster_namespace_deployment:actual_replicas:count
|
|
|
|
|
*
|
2021-04-01 03:34:23 +00:00
|
|
|
quantile_over_time(0.99, cluster_namespace_deployment:container_cpu_usage_seconds_total:sum_rate[24h])
|
2021-03-20 03:27:27 +00:00
|
|
|
/
|
2021-04-01 03:34:23 +00:00
|
|
|
cluster_namespace_deployment:kube_pod_container_resource_requests_cpu_cores:sum
|
2021-03-20 03:27:27 +00:00
|
|
|
)
|
|
|
|
|
labels:
|
|
|
|
|
reason: cpu_usage
|
|
|
|
|
record: cluster_namespace_deployment_reason:required_replicas:count
|
2021-04-01 03:34:23 +00:00
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, namespace, deployment) (
|
|
|
|
|
label_replace(
|
2021-07-29 03:25:46 +00:00
|
|
|
label_replace(
|
|
|
|
|
container_memory_usage_bytes,
|
|
|
|
|
"deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
|
|
|
|
|
),
|
|
|
|
|
# The question mark in "(.*?)" is used to make it non-greedy, otherwise it
|
|
|
|
|
# always matches everything and the (optional) zone is not removed.
|
|
|
|
|
"deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
|
2021-04-01 03:34:23 +00:00
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
record: cluster_namespace_deployment:container_memory_usage_bytes:sum
|
|
|
|
|
- expr: |
|
2021-06-02 05:48:33 +00:00
|
|
|
# This recording rule is made compatible with the breaking changes introduced in kube-state-metrics v2
|
|
|
|
|
# that remove resource metrics, ref:
|
|
|
|
|
# - https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
|
|
|
|
|
# - https://github.com/kubernetes/kube-state-metrics/pull/1004
|
|
|
|
|
#
|
|
|
|
|
# This is the old expression, compatible with kube-state-metrics < v2.0.0,
|
|
|
|
|
# where kube_pod_container_resource_requests_memory_bytes was removed:
|
|
|
|
|
(
|
|
|
|
|
sum by (cluster, namespace, deployment) (
|
|
|
|
|
label_replace(
|
2021-07-29 03:25:46 +00:00
|
|
|
label_replace(
|
|
|
|
|
kube_pod_container_resource_requests_memory_bytes,
|
|
|
|
|
"deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
|
|
|
|
|
),
|
|
|
|
|
# The question mark in "(.*?)" is used to make it non-greedy, otherwise it
|
|
|
|
|
# always matches everything and the (optional) zone is not removed.
|
|
|
|
|
"deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
|
2021-06-02 05:48:33 +00:00
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
or
|
|
|
|
|
# This expression is compatible with kube-state-metrics >= v1.4.0,
|
|
|
|
|
# where kube_pod_container_resource_requests was introduced.
|
|
|
|
|
(
|
|
|
|
|
sum by (cluster, namespace, deployment) (
|
|
|
|
|
label_replace(
|
2021-07-29 03:25:46 +00:00
|
|
|
label_replace(
|
|
|
|
|
kube_pod_container_resource_requests{resource="memory"},
|
|
|
|
|
"deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))"
|
|
|
|
|
),
|
|
|
|
|
# The question mark in "(.*?)" is used to make it non-greedy, otherwise it
|
|
|
|
|
# always matches everything and the (optional) zone is not removed.
|
|
|
|
|
"deployment", "$1", "deployment", "(.*?)(?:-zone-[a-z])?"
|
2021-06-02 05:48:33 +00:00
|
|
|
)
|
2021-04-01 03:34:23 +00:00
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
record: cluster_namespace_deployment:kube_pod_container_resource_requests_memory_bytes:sum
|
2021-03-20 03:27:27 +00:00
|
|
|
- expr: |
|
|
|
|
|
ceil(
|
|
|
|
|
cluster_namespace_deployment:actual_replicas:count
|
|
|
|
|
*
|
2021-04-01 03:34:23 +00:00
|
|
|
quantile_over_time(0.99, cluster_namespace_deployment:container_memory_usage_bytes:sum[24h])
|
2021-03-20 03:27:27 +00:00
|
|
|
/
|
2021-04-01 03:34:23 +00:00
|
|
|
cluster_namespace_deployment:kube_pod_container_resource_requests_memory_bytes:sum
|
2021-03-20 03:27:27 +00:00
|
|
|
)
|
|
|
|
|
labels:
|
|
|
|
|
reason: memory_usage
|
|
|
|
|
record: cluster_namespace_deployment_reason:required_replicas:count
|
2021-09-23 03:23:16 +00:00
|
|
|
- name: cortex_alertmanager_rules
|
|
|
|
|
rules:
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, job, pod) (cortex_alertmanager_alerts)
|
|
|
|
|
record: cluster_job_pod:cortex_alertmanager_alerts:sum
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, job, pod) (cortex_alertmanager_silences)
|
|
|
|
|
record: cluster_job_pod:cortex_alertmanager_silences:sum
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, job) (rate(cortex_alertmanager_alerts_received_total[5m]))
|
|
|
|
|
record: cluster_job:cortex_alertmanager_alerts_received_total:rate5m
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, job) (rate(cortex_alertmanager_alerts_invalid_total[5m]))
|
|
|
|
|
record: cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_total[5m]))
|
|
|
|
|
record: cluster_job_integration:cortex_alertmanager_notifications_total:rate5m
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, job, integration) (rate(cortex_alertmanager_notifications_failed_total[5m]))
|
|
|
|
|
record: cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, job) (rate(cortex_alertmanager_state_replication_total[5m]))
|
|
|
|
|
record: cluster_job:cortex_alertmanager_state_replication_total:rate5m
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, job) (rate(cortex_alertmanager_state_replication_failed_total[5m]))
|
|
|
|
|
record: cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_total[5m]))
|
|
|
|
|
record: cluster_job:cortex_alertmanager_partial_state_merges_total:rate5m
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, job) (rate(cortex_alertmanager_partial_state_merges_failed_total[5m]))
|
|
|
|
|
record: cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m
|
