mirrors/monitoring-mixins-website
1
0
Fork 0
mirror of https://github.com/monitoring-mixins/website.git synced 2024-12-15 17:50:48 +00:00
Code Activity

assets,site/content: daily assets regeneration

Browse source
This commit is contained in:
github-actions[bot] 2021-07-01 03:25:31 +00:00
parent 57aafaaeb8
commit a1b070c941
12 changed files with 265 additions and 265 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
assets/cortex/dashboards/alertmanager.json
View file

@ -224,7 +224,7 @@
{
"expr": "sum(rate(cortex_alertmanager_alerts_received_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_alerts_invalid_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "success",
"legendLink": null,
@ -233,7 +233,7 @@
{
"expr": "sum(rate(cortex_alertmanager_alerts_invalid_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "failed",
"legendLink": null,
@ -322,7 +322,7 @@
{
"expr": "sum(rate(cortex_alertmanager_notifications_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "success",
"legendLink": null,
@ -331,7 +331,7 @@
{
"expr": "sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "failed",
"legendLink": null,
@ -408,7 +408,7 @@
{
"expr": "(\nsum(rate(cortex_alertmanager_notifications_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) by(integration)\n-\nsum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) by(integration)\n) > 0\nor on () vector(0)\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "success - {{ integration }}",
"legendLink": null,
@ -417,7 +417,7 @@
{
"expr": "sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) by(integration)",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "failed - {{ integration }}",
"legendLink": null,
@ -494,7 +494,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -503,7 +503,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -512,7 +512,7 @@
{
"expr": "sum(rate(cortex_alertmanager_notification_latency_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval])) * 1e3 / sum(rate(cortex_alertmanager_notification_latency_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -609,7 +609,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_v1_alerts|alertmanager\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",

20
assets/cortex/dashboards/cortex-compactor-resources.json
View file

@ -66,7 +66,7 @@
{
"expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"compactor\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{pod}}",
"legendLink": null,
@ -75,7 +75,7 @@
{
"expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"compactor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"compactor\"})",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "limit",
"legendLink": null,
@ -156,7 +156,7 @@
{
"expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"compactor\"})",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{pod}}",
"legendLink": null,
@ -165,7 +165,7 @@
{
"expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"compactor\"} > 0)",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "limit",
"legendLink": null,
@ -240,7 +240,7 @@
{
"expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"})",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{pod}}",
"legendLink": null,
@ -327,7 +327,7 @@
{
"expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{pod}}",
"legendLink": null,
@ -404,7 +404,7 @@
{
"expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{pod}}",
"legendLink": null,
@ -493,7 +493,7 @@
{
"expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=\"compactor\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{pod}} - {{device}}",
"legendLink": null,
@ -570,7 +570,7 @@
{
"expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=\"compactor\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{pod}} - {{device}}",
"legendLink": null,
@ -647,7 +647,7 @@
{
"expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=\"compactor\"\n }\n)\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{persistentvolumeclaim}}",
"legendLink": null,

76
assets/cortex/dashboards/cortex-compactor.json
View file

@ -65,7 +65,7 @@
{
"expr": "sum(rate(cortex_compactor_runs_started_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "started",
"legendLink": null,
@ -74,7 +74,7 @@
{
"expr": "sum(rate(cortex_compactor_runs_completed_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "completed",
"legendLink": null,
@ -83,7 +83,7 @@
{
"expr": "sum(rate(cortex_compactor_runs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "failed",
"legendLink": null,
@ -161,7 +161,7 @@
{
"expr": "(\n cortex_compactor_tenants_processing_succeeded{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"} +\n cortex_compactor_tenants_processing_failed{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"} +\n cortex_compactor_tenants_skipped{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}\n) / cortex_compactor_tenants_discovered{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{pod}}",
"legendLink": null,
@ -251,7 +251,7 @@
{
"expr": "sum(rate(prometheus_tsdb_compactions_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "blocks",
"legendLink": null,
@ -329,7 +329,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -338,7 +338,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -347,7 +347,7 @@
{
"expr": "sum(rate(prometheus_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) * 1e3 / sum(rate(prometheus_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -436,7 +436,7 @@
{
"expr": "avg(max by(user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "avg",
"legendLink": null,
@ -514,7 +514,7 @@
{
"expr": "topk(10, max by(user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{user}}",
"legendLink": null,
@ -603,7 +603,7 @@
{
"expr": "sum(rate(cortex_compactor_blocks_marked_for_deletion_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "blocks",
"legendLink": null,
@ -683,7 +683,7 @@
{
"expr": "sum(rate(cortex_compactor_blocks_cleaned_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "successful",
"legendLink": null,
@ -692,7 +692,7 @@
{
"expr": "sum(rate(cortex_compactor_block_cleanup_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "failed",
"legendLink": null,
@ -784,7 +784,7 @@
{
"expr": "sum(rate(cortex_compactor_meta_syncs_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) - sum(rate(cortex_compactor_meta_sync_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "successful",
"legendLink": null,
@ -793,7 +793,7 @@
{
"expr": "sum(rate(cortex_compactor_meta_sync_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "failed",
"legendLink": null,
@ -870,7 +870,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -879,7 +879,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -888,7 +888,7 @@
{
"expr": "sum(rate(cortex_compactor_meta_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) * 1e3 / sum(rate(cortex_compactor_meta_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -977,7 +977,7 @@
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{operation}}",
"legendLink": null,
@ -1054,7 +1054,7 @@
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{operation}}",
"legendLink": null,
@ -1131,7 +1131,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -1140,7 +1140,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -1149,7 +1149,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -1226,7 +1226,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -1235,7 +1235,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -1244,7 +1244,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -1333,7 +1333,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -1342,7 +1342,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -1351,7 +1351,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -1428,7 +1428,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -1437,7 +1437,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -1446,7 +1446,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -1523,7 +1523,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -1532,7 +1532,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -1541,7 +1541,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -1618,7 +1618,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -1627,7 +1627,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -1636,7 +1636,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",

4
assets/cortex/dashboards/cortex-config.json
View file

@ -60,7 +60,7 @@
{
"expr": "count(cortex_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}) by (sha256)",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "sha256:{{sha256}}",
"legendLink": null,
@ -149,7 +149,7 @@
{
"expr": "count(cortex_runtime_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}) by (sha256)",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "sha256:{{sha256}}",
"legendLink": null,

44
assets/cortex/dashboards/cortex-object-store.json
View file

@ -60,7 +60,7 @@
{
"expr": "sum by(component) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{component}}",
"legendLink": null,
@ -137,7 +137,7 @@
{
"expr": "sum by(component) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by(component) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{component}}",
"legendLink": null,
@ -226,7 +226,7 @@
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{operation}}",
"legendLink": null,
@ -303,7 +303,7 @@
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{operation}}",
"legendLink": null,
@ -392,7 +392,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -401,7 +401,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -410,7 +410,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -487,7 +487,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -496,7 +496,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -505,7 +505,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -582,7 +582,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -591,7 +591,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -600,7 +600,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -689,7 +689,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -698,7 +698,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -707,7 +707,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -784,7 +784,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -793,7 +793,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -802,7 +802,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -879,7 +879,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -888,7 +888,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -897,7 +897,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",

106
assets/cortex/dashboards/cortex-queries.json
View file

@ -60,7 +60,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -69,7 +69,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -78,7 +78,7 @@
{
"expr": "sum(rate(cortex_query_frontend_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_query_frontend_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -155,7 +155,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval])) by (le)) * 1",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -164,7 +164,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval])) by (le)) * 1",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -173,7 +173,7 @@
{
"expr": "sum(rate(cortex_query_frontend_retries_sum{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_query_frontend_retries_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -250,7 +250,7 @@
{
"expr": "cortex_query_frontend_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{cluster}} / {{namespace}} / {{pod}}",
"legendLink": null,
@ -339,7 +339,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -348,7 +348,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -357,7 +357,7 @@
{
"expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -434,7 +434,7 @@
{
"expr": "cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{cluster}} / {{namespace}} / {{pod}}",
"legendLink": null,
@ -523,7 +523,7 @@
{
"expr": "sum(rate(cortex_cache_hits{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m])) / sum(rate(cortex_cache_fetched_keys{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Hit Rate",
"legendLink": null,
@ -600,7 +600,7 @@
{
"expr": "sum(rate(cortex_cache_fetched_keys{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m])) - sum(rate(cortex_cache_hits{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Miss Rate",
"legendLink": null,
@ -689,7 +689,7 @@
{
"expr": "sum(rate(cortex_frontend_split_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m])) / sum(rate(cortex_frontend_query_range_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\", method=\"split_by_interval\"}[1m]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "partition rate",
"legendLink": null,
@ -766,7 +766,7 @@
{
"expr": "sum(rate(cortex_frontend_mapped_asts_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m])) / sum(rate(cortex_frontend_split_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m])) * 100",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "shard rate",
"legendLink": null,
@ -843,7 +843,7 @@
{
"expr": "sum(rate(cortex_frontend_sharded_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m])) / sum(rate(cortex_frontend_mapped_asts_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"legendLink": null,
@ -932,7 +932,7 @@
{
"expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{slice}}",
"legendLink": null,
@ -1009,7 +1009,7 @@
{
"expr": "sum(rate(cortex_cache_fetched_keys{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",name=\"chunksmemcache\"}[1m])) - sum(rate(cortex_cache_hits{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",name=\"chunksmemcache\"}[1m]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Hit rate",
"legendLink": null,
@ -1086,7 +1086,7 @@
{
"expr": "sum(rate(cortex_cache_corrupt_chunks_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[1m]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Corrupt chunks",
"legendLink": null,
@ -1463,7 +1463,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) by (le)) * 1",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -1472,7 +1472,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) by (le)) * 1",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -1481,7 +1481,7 @@
{
"expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -1558,7 +1558,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) by (le)) * 1",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -1567,7 +1567,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) by (le)) * 1",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -1576,7 +1576,7 @@
{
"expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -1653,7 +1653,7 @@
{
"expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[1m])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[1m]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Failure Rate",
"legendLink": null,
@ -1742,7 +1742,7 @@
{
"expr": "max(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"})",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Max",
"legendLink": null,
@ -1751,7 +1751,7 @@
{
"expr": "min(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"})",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Min",
"legendLink": null,
@ -1760,7 +1760,7 @@
{
"expr": "avg(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"})",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"legendLink": null,
@ -1840,7 +1840,7 @@
{
"expr": "sum(rate(cortex_bucket_index_loads_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_bucket_index_load_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "successful",
"legendLink": null,
@ -1849,7 +1849,7 @@
{
"expr": "sum(rate(cortex_bucket_index_load_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "failed",
"legendLink": null,
@ -1926,7 +1926,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -1935,7 +1935,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -1944,7 +1944,7 @@
{
"expr": "sum(rate(cortex_bucket_index_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_bucket_index_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -2033,7 +2033,7 @@
{
"expr": "sum(rate(cortex_bucket_store_series_blocks_queried_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "blocks",
"legendLink": null,
@ -2110,7 +2110,7 @@
{
"expr": "sum by(data_type) (rate(cortex_bucket_store_series_data_fetched_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{data_type}}",
"legendLink": null,
@ -2187,7 +2187,7 @@
{
"expr": "sum by(data_type) (rate(cortex_bucket_store_series_data_touched_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{data_type}}",
"legendLink": null,
@ -2276,7 +2276,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_series_get_all_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -2285,7 +2285,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_series_get_all_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -2294,7 +2294,7 @@
{
"expr": "sum(rate(cortex_bucket_store_series_get_all_duration_seconds_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_bucket_store_series_get_all_duration_seconds_count{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -2371,7 +2371,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_series_merge_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -2380,7 +2380,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_series_merge_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -2389,7 +2389,7 @@
{
"expr": "sum(rate(cortex_bucket_store_series_merge_duration_seconds_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_bucket_store_series_merge_duration_seconds_count{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -2466,7 +2466,7 @@
{
"expr": "sum(rate(cortex_bucket_store_series_result_series_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) / sum(rate(cortex_bucket_store_series_result_series_count{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "avg series returned",
"legendLink": null,
@ -2555,7 +2555,7 @@
{
"expr": "cortex_bucket_store_blocks_loaded{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{pod}}",
"legendLink": null,
@ -2635,7 +2635,7 @@
{
"expr": "sum(rate(cortex_bucket_store_block_loads_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "successful",
"legendLink": null,
@ -2644,7 +2644,7 @@
{
"expr": "sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "failed",
"legendLink": null,
@ -2724,7 +2724,7 @@
{
"expr": "sum(rate(cortex_bucket_store_block_drops_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "successful",
"legendLink": null,
@ -2733,7 +2733,7 @@
{
"expr": "sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "failed",
"legendLink": null,
@ -2822,7 +2822,7 @@
{
"expr": "cortex_bucket_store_indexheader_lazy_load_total{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"} - cortex_bucket_store_indexheader_lazy_unload_total{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{pod}}",
"legendLink": null,
@ -2899,7 +2899,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -2908,7 +2908,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -2917,7 +2917,7 @@
{
"expr": "sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",

150
assets/cortex/dashboards/cortex-reads.json
View file

@ -257,7 +257,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -420,7 +420,7 @@
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "",
"legendLink": null,
@ -515,7 +515,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -678,7 +678,7 @@
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "",
"legendLink": null,
@ -784,7 +784,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -861,7 +861,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -870,7 +870,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -879,7 +879,7 @@
{
"expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -976,7 +976,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_cache_request_duration_seconds_count{method=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -1165,7 +1165,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -1328,7 +1328,7 @@
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "",
"legendLink": null,
@ -1423,7 +1423,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -1586,7 +1586,7 @@
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval])))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "",
"legendLink": null,
@ -1681,7 +1681,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",route=~\"/gatewaypb.StoreGateway/.*\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -1844,7 +1844,7 @@
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\", route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval])))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "",
"legendLink": null,
@ -1931,7 +1931,7 @@
{
"expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n component=\"store-gateway\",\n name=\"index-cache\",\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{operation}}",
"legendLink": null,
@ -2008,7 +2008,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -2017,7 +2017,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -2026,7 +2026,7 @@
{
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -2104,7 +2104,7 @@
{
"expr": "sum by(item_type) (\n rate(\n thanos_store_index_cache_hits_total{\n component=\"store-gateway\",\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"\n }[$__rate_interval]\n )\n)\n/\nsum by(item_type) (\n rate(\n thanos_store_index_cache_requests_total{\n component=\"store-gateway\",\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{item_type}}",
"legendLink": null,
@ -2193,7 +2193,7 @@
{
"expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{operation}}",
"legendLink": null,
@ -2270,7 +2270,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -2279,7 +2279,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -2288,7 +2288,7 @@
{
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -2365,7 +2365,7 @@
{
"expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "items",
"legendLink": null,
@ -2454,7 +2454,7 @@
{
"expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{operation}}",
"legendLink": null,
@ -2531,7 +2531,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -2540,7 +2540,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -2549,7 +2549,7 @@
{
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -2626,7 +2626,7 @@
{
"expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "items",
"legendLink": null,
@ -2715,7 +2715,7 @@
{
"expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{operation}}",
"legendLink": null,
@ -2792,7 +2792,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -2801,7 +2801,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -2810,7 +2810,7 @@
{
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -2887,7 +2887,7 @@
{
"expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "items",
"legendLink": null,
@ -2976,7 +2976,7 @@
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{operation}}",
"legendLink": null,
@ -3053,7 +3053,7 @@
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{operation}}",
"legendLink": null,
@ -3130,7 +3130,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -3139,7 +3139,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -3148,7 +3148,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -3225,7 +3225,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -3234,7 +3234,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -3243,7 +3243,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -3332,7 +3332,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -3341,7 +3341,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -3350,7 +3350,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -3427,7 +3427,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -3436,7 +3436,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -3445,7 +3445,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -3522,7 +3522,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -3531,7 +3531,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -3540,7 +3540,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -3617,7 +3617,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -3626,7 +3626,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -3635,7 +3635,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -3724,7 +3724,7 @@
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{operation}}",
"legendLink": null,
@ -3801,7 +3801,7 @@
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{operation}}",
"legendLink": null,
@ -3878,7 +3878,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -3887,7 +3887,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -3896,7 +3896,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -3973,7 +3973,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -3982,7 +3982,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -3991,7 +3991,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -4080,7 +4080,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -4089,7 +4089,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -4098,7 +4098,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -4175,7 +4175,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -4184,7 +4184,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -4193,7 +4193,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -4270,7 +4270,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -4279,7 +4279,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -4288,7 +4288,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -4365,7 +4365,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -4374,7 +4374,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -4383,7 +4383,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",

4
assets/cortex/dashboards/cortex-rollout-progress.json
View file

@ -1253,7 +1253,7 @@
{
"expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}))[1h:])\n)\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "writes",
"legendLink": null,
@ -1262,7 +1262,7 @@
{
"expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}))[1h:])\n)\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "reads",
"legendLink": null,

50
assets/cortex/dashboards/cortex-writes.json
View file

@ -407,7 +407,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -570,7 +570,7 @@
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}[$__rate_interval])))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "",
"legendLink": null,
@ -665,7 +665,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -828,7 +828,7 @@
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__rate_interval])))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "",
"legendLink": null,
@ -923,7 +923,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -1112,7 +1112,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",route=\"/cortex.Ingester/Push\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -1275,7 +1275,7 @@
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\", route=\"/cortex.Ingester/Push\"}[$__rate_interval])))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "",
"legendLink": null,
@ -1370,7 +1370,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -1555,7 +1555,7 @@
{
"expr": "sum(rate(cortex_ingester_shipper_uploads_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_ingester_shipper_upload_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "successful",
"legendLink": null,
@ -1564,7 +1564,7 @@
{
"expr": "sum(rate(cortex_ingester_shipper_upload_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "failed",
"legendLink": null,
@ -1642,7 +1642,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -1651,7 +1651,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -1660,7 +1660,7 @@
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -1753,7 +1753,7 @@
{
"expr": "sum(rate(cortex_ingester_tsdb_compactions_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "successful",
"legendLink": null,
@ -1762,7 +1762,7 @@
{
"expr": "sum(rate(cortex_ingester_tsdb_compactions_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "failed",
"legendLink": null,
@ -1840,7 +1840,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -1849,7 +1849,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -1858,7 +1858,7 @@
{
"expr": "sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -1951,7 +1951,7 @@
{
"expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "successful",
"legendLink": null,
@ -1960,7 +1960,7 @@
{
"expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "failed",
"legendLink": null,
@ -2041,7 +2041,7 @@
{
"expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "successful",
"legendLink": null,
@ -2050,7 +2050,7 @@
{
"expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "failed",
"legendLink": null,
@ -2128,7 +2128,7 @@
{
"expr": "sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) / sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "avg",
"legendLink": null,
@ -2208,7 +2208,7 @@
{
"expr": "sum(rate(cortex_ingester_wal_corruptions_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "WAL",
"legendLink": null,
@ -2217,7 +2217,7 @@
{
"expr": "sum(rate(cortex_ingester_tsdb_mmap_chunk_corruptions_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "mmap-ed chunks",
"legendLink": null,

52
assets/cortex/dashboards/ruler.json
View file

@ -376,7 +376,7 @@
{
"expr": "sum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n-\nsum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "success",
"legendLink": null,
@ -385,7 +385,7 @@
{
"expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "failed",
"legendLink": null,
@ -462,7 +462,7 @@
{
"expr": "sum (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n /\nsum (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "average",
"legendLink": null,
@ -559,7 +559,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -748,7 +748,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -825,7 +825,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -834,7 +834,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -843,7 +843,7 @@
{
"expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -940,7 +940,7 @@
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{status}}",
"refId": "A",
@ -1017,7 +1017,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -1026,7 +1026,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -1035,7 +1035,7 @@
{
"expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -1124,7 +1124,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) by (le)) * 1",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -1133,7 +1133,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) by (le)) * 1",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -1142,7 +1142,7 @@
{
"expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -1219,7 +1219,7 @@
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) by (le)) * 1",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "99th Percentile",
"refId": "A",
@ -1228,7 +1228,7 @@
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) by (le)) * 1",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "50th Percentile",
"refId": "B",
@ -1237,7 +1237,7 @@
{
"expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Average",
"refId": "C",
@ -1314,7 +1314,7 @@
{
"expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[1m])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[1m]))",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "Failure Rate",
"legendLink": null,
@ -1403,7 +1403,7 @@
{
"expr": "sum by(user) (rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n> 0\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{ user }}",
"legendLink": null,
@ -1480,7 +1480,7 @@
{
"expr": "sum by(user) (rate(cortex_prometheus_notifications_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_queue_capacity{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval])) > 0\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{ user }}",
"legendLink": null,
@ -1557,7 +1557,7 @@
{
"expr": "sum by (user) (increase(cortex_prometheus_notifications_dropped_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval])) > 0\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{ user }}",
"legendLink": null,
@ -1646,7 +1646,7 @@
{
"expr": "sum by(user) (rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval])) > 0",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{ user }}",
"legendLink": null,
@ -1723,7 +1723,7 @@
{
"expr": "rate(cortex_prometheus_rule_group_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval])\n /\nrate(cortex_prometheus_rule_group_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval])\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{ user }}",
"legendLink": null,
@ -1800,7 +1800,7 @@
{
"expr": "sum by(rule_group) (rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval])) > 0",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{ rule_group }}",
"legendLink": null,
@ -1889,7 +1889,7 @@
{
"expr": "sum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n",
"format": "time_series",
"interval": "1m",
"interval": "15s",
"intervalFactor": 2,
"legendFormat": "{{ user }}",
"legendLink": null,

2
assets/kubernetes/alerts.yaml
View file

@ -54,7 +54,7 @@ groups:
expr: |
(
kube_deployment_spec_replicas{job="kube-state-metrics"}
!=
>
kube_deployment_status_replicas_available{job="kube-state-metrics"}
) and (
changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m])

2
site/content/kubernetes/_index.md
View file

@ -93,7 +93,7 @@ annotations:
expr: |
(
kube_deployment_spec_replicas{job="kube-state-metrics"}
!=
>
kube_deployment_status_replicas_available{job="kube-state-metrics"}
) and (
changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m])