From a1b070c941d01b122942b13457ad76545ea3fc8a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 1 Jul 2021 03:25:31 +0000 Subject: [PATCH] assets,site/content: daily assets regeneration --- assets/cortex/dashboards/alertmanager.json | 20 +-- .../cortex-compactor-resources.json | 20 +-- .../cortex/dashboards/cortex-compactor.json | 76 ++++----- assets/cortex/dashboards/cortex-config.json | 4 +- .../dashboards/cortex-object-store.json | 44 ++--- assets/cortex/dashboards/cortex-queries.json | 106 ++++++------- assets/cortex/dashboards/cortex-reads.json | 150 +++++++++--------- .../dashboards/cortex-rollout-progress.json | 4 +- assets/cortex/dashboards/cortex-writes.json | 50 +++--- assets/cortex/dashboards/ruler.json | 52 +++--- assets/kubernetes/alerts.yaml | 2 +- site/content/kubernetes/_index.md | 2 +- 12 files changed, 265 insertions(+), 265 deletions(-) diff --git a/assets/cortex/dashboards/alertmanager.json b/assets/cortex/dashboards/alertmanager.json index 6b71cbe..8bd0caa 100644 --- a/assets/cortex/dashboards/alertmanager.json +++ b/assets/cortex/dashboards/alertmanager.json @@ -224,7 +224,7 @@ { "expr": "sum(rate(cortex_alertmanager_alerts_received_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_alerts_invalid_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "success", "legendLink": null, @@ -233,7 +233,7 @@ { "expr": "sum(rate(cortex_alertmanager_alerts_invalid_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "failed", "legendLink": null, @@ -322,7 +322,7 @@ { "expr": "sum(rate(cortex_alertmanager_notifications_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "success", "legendLink": null, @@ -331,7 +331,7 @@ { "expr": "sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "failed", "legendLink": null, @@ -408,7 +408,7 @@ { "expr": "(\nsum(rate(cortex_alertmanager_notifications_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) by(integration)\n-\nsum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) by(integration)\n) > 0\nor on () vector(0)\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "success - {{ integration }}", "legendLink": null, @@ -417,7 +417,7 @@ { "expr": "sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) by(integration)", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "failed - {{ integration }}", "legendLink": null, @@ -494,7 +494,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -503,7 +503,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -512,7 +512,7 @@ { "expr": "sum(rate(cortex_alertmanager_notification_latency_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval])) * 1e3 / sum(rate(cortex_alertmanager_notification_latency_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -609,7 +609,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_v1_alerts|alertmanager\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", diff --git a/assets/cortex/dashboards/cortex-compactor-resources.json b/assets/cortex/dashboards/cortex-compactor-resources.json index e552933..cb8a5f3 100644 --- a/assets/cortex/dashboards/cortex-compactor-resources.json +++ b/assets/cortex/dashboards/cortex-compactor-resources.json @@ -66,7 +66,7 @@ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"compactor\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{pod}}", "legendLink": null, @@ -75,7 +75,7 @@ { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"compactor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"compactor\"})", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "limit", "legendLink": null, @@ -156,7 +156,7 @@ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"compactor\"})", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{pod}}", "legendLink": null, @@ -165,7 +165,7 @@ { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"compactor\"} > 0)", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "limit", "legendLink": null, @@ -240,7 +240,7 @@ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"})", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{pod}}", "legendLink": null, @@ -327,7 +327,7 @@ { "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"compactor.*\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{pod}}", "legendLink": null, @@ -404,7 +404,7 @@ { "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"compactor.*\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{pod}}", "legendLink": null, @@ -493,7 +493,7 @@ { "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=\"compactor\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", "legendLink": null, @@ -570,7 +570,7 @@ { "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=\"compactor\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", "legendLink": null, @@ -647,7 +647,7 @@ { "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=\"compactor\"\n }\n)\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{persistentvolumeclaim}}", "legendLink": null, diff --git a/assets/cortex/dashboards/cortex-compactor.json b/assets/cortex/dashboards/cortex-compactor.json index fea8db1..d8efa13 100644 --- a/assets/cortex/dashboards/cortex-compactor.json +++ b/assets/cortex/dashboards/cortex-compactor.json @@ -65,7 +65,7 @@ { "expr": "sum(rate(cortex_compactor_runs_started_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "started", "legendLink": null, @@ -74,7 +74,7 @@ { "expr": "sum(rate(cortex_compactor_runs_completed_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "completed", "legendLink": null, @@ -83,7 +83,7 @@ { "expr": "sum(rate(cortex_compactor_runs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "failed", "legendLink": null, @@ -161,7 +161,7 @@ { "expr": "(\n cortex_compactor_tenants_processing_succeeded{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"} +\n cortex_compactor_tenants_processing_failed{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"} +\n cortex_compactor_tenants_skipped{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}\n) / cortex_compactor_tenants_discovered{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{pod}}", "legendLink": null, @@ -251,7 +251,7 @@ { "expr": "sum(rate(prometheus_tsdb_compactions_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "blocks", "legendLink": null, @@ -329,7 +329,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -338,7 +338,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -347,7 +347,7 @@ { "expr": "sum(rate(prometheus_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) * 1e3 / sum(rate(prometheus_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -436,7 +436,7 @@ { "expr": "avg(max by(user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "avg", "legendLink": null, @@ -514,7 +514,7 @@ { "expr": "topk(10, max by(user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{user}}", "legendLink": null, @@ -603,7 +603,7 @@ { "expr": "sum(rate(cortex_compactor_blocks_marked_for_deletion_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "blocks", "legendLink": null, @@ -683,7 +683,7 @@ { "expr": "sum(rate(cortex_compactor_blocks_cleaned_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "successful", "legendLink": null, @@ -692,7 +692,7 @@ { "expr": "sum(rate(cortex_compactor_block_cleanup_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "failed", "legendLink": null, @@ -784,7 +784,7 @@ { "expr": "sum(rate(cortex_compactor_meta_syncs_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) - sum(rate(cortex_compactor_meta_sync_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "successful", "legendLink": null, @@ -793,7 +793,7 @@ { "expr": "sum(rate(cortex_compactor_meta_sync_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "failed", "legendLink": null, @@ -870,7 +870,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -879,7 +879,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -888,7 +888,7 @@ { "expr": "sum(rate(cortex_compactor_meta_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) * 1e3 / sum(rate(cortex_compactor_meta_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -977,7 +977,7 @@ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{operation}}", "legendLink": null, @@ -1054,7 +1054,7 @@ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{operation}}", "legendLink": null, @@ -1131,7 +1131,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -1140,7 +1140,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -1149,7 +1149,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -1226,7 +1226,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -1235,7 +1235,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -1244,7 +1244,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -1333,7 +1333,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -1342,7 +1342,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -1351,7 +1351,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -1428,7 +1428,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -1437,7 +1437,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -1446,7 +1446,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -1523,7 +1523,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -1532,7 +1532,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -1541,7 +1541,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -1618,7 +1618,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -1627,7 +1627,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -1636,7 +1636,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", diff --git a/assets/cortex/dashboards/cortex-config.json b/assets/cortex/dashboards/cortex-config.json index 9ee141e..1ef44b6 100644 --- a/assets/cortex/dashboards/cortex-config.json +++ b/assets/cortex/dashboards/cortex-config.json @@ -60,7 +60,7 @@ { "expr": "count(cortex_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}) by (sha256)", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "sha256:{{sha256}}", "legendLink": null, @@ -149,7 +149,7 @@ { "expr": "count(cortex_runtime_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}) by (sha256)", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "sha256:{{sha256}}", "legendLink": null, diff --git a/assets/cortex/dashboards/cortex-object-store.json b/assets/cortex/dashboards/cortex-object-store.json index 62185be..2734fe5 100644 --- a/assets/cortex/dashboards/cortex-object-store.json +++ b/assets/cortex/dashboards/cortex-object-store.json @@ -60,7 +60,7 @@ { "expr": "sum by(component) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{component}}", "legendLink": null, @@ -137,7 +137,7 @@ { "expr": "sum by(component) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by(component) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{component}}", "legendLink": null, @@ -226,7 +226,7 @@ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{operation}}", "legendLink": null, @@ -303,7 +303,7 @@ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{operation}}", "legendLink": null, @@ -392,7 +392,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -401,7 +401,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -410,7 +410,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -487,7 +487,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -496,7 +496,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -505,7 +505,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -582,7 +582,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -591,7 +591,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -600,7 +600,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -689,7 +689,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -698,7 +698,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -707,7 +707,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -784,7 +784,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -793,7 +793,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -802,7 +802,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -879,7 +879,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -888,7 +888,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -897,7 +897,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", diff --git a/assets/cortex/dashboards/cortex-queries.json b/assets/cortex/dashboards/cortex-queries.json index 580d75d..a0f0ec8 100644 --- a/assets/cortex/dashboards/cortex-queries.json +++ b/assets/cortex/dashboards/cortex-queries.json @@ -60,7 +60,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -69,7 +69,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -78,7 +78,7 @@ { "expr": "sum(rate(cortex_query_frontend_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_query_frontend_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -155,7 +155,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval])) by (le)) * 1", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -164,7 +164,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval])) by (le)) * 1", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -173,7 +173,7 @@ { "expr": "sum(rate(cortex_query_frontend_retries_sum{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_query_frontend_retries_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -250,7 +250,7 @@ { "expr": "cortex_query_frontend_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{cluster}} / {{namespace}} / {{pod}}", "legendLink": null, @@ -339,7 +339,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -348,7 +348,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -357,7 +357,7 @@ { "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -434,7 +434,7 @@ { "expr": "cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{cluster}} / {{namespace}} / {{pod}}", "legendLink": null, @@ -523,7 +523,7 @@ { "expr": "sum(rate(cortex_cache_hits{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m])) / sum(rate(cortex_cache_fetched_keys{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Hit Rate", "legendLink": null, @@ -600,7 +600,7 @@ { "expr": "sum(rate(cortex_cache_fetched_keys{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m])) - sum(rate(cortex_cache_hits{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Miss Rate", "legendLink": null, @@ -689,7 +689,7 @@ { "expr": "sum(rate(cortex_frontend_split_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m])) / sum(rate(cortex_frontend_query_range_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\", method=\"split_by_interval\"}[1m]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "partition rate", "legendLink": null, @@ -766,7 +766,7 @@ { "expr": "sum(rate(cortex_frontend_mapped_asts_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m])) / sum(rate(cortex_frontend_split_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m])) * 100", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "shard rate", "legendLink": null, @@ -843,7 +843,7 @@ { "expr": "sum(rate(cortex_frontend_sharded_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m])) / sum(rate(cortex_frontend_mapped_asts_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[1m]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "legendLink": null, @@ -932,7 +932,7 @@ { "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{slice}}", "legendLink": null, @@ -1009,7 +1009,7 @@ { "expr": "sum(rate(cortex_cache_fetched_keys{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",name=\"chunksmemcache\"}[1m])) - sum(rate(cortex_cache_hits{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",name=\"chunksmemcache\"}[1m]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Hit rate", "legendLink": null, @@ -1086,7 +1086,7 @@ { "expr": "sum(rate(cortex_cache_corrupt_chunks_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[1m]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Corrupt chunks", "legendLink": null, @@ -1463,7 +1463,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) by (le)) * 1", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -1472,7 +1472,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) by (le)) * 1", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -1481,7 +1481,7 @@ { "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -1558,7 +1558,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) by (le)) * 1", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -1567,7 +1567,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) by (le)) * 1", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -1576,7 +1576,7 @@ { "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -1653,7 +1653,7 @@ { "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[1m])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[1m]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Failure Rate", "legendLink": null, @@ -1742,7 +1742,7 @@ { "expr": "max(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"})", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Max", "legendLink": null, @@ -1751,7 +1751,7 @@ { "expr": "min(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"})", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Min", "legendLink": null, @@ -1760,7 +1760,7 @@ { "expr": "avg(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"})", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "legendLink": null, @@ -1840,7 +1840,7 @@ { "expr": "sum(rate(cortex_bucket_index_loads_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_bucket_index_load_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "successful", "legendLink": null, @@ -1849,7 +1849,7 @@ { "expr": "sum(rate(cortex_bucket_index_load_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "failed", "legendLink": null, @@ -1926,7 +1926,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -1935,7 +1935,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -1944,7 +1944,7 @@ { "expr": "sum(rate(cortex_bucket_index_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_bucket_index_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -2033,7 +2033,7 @@ { "expr": "sum(rate(cortex_bucket_store_series_blocks_queried_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "blocks", "legendLink": null, @@ -2110,7 +2110,7 @@ { "expr": "sum by(data_type) (rate(cortex_bucket_store_series_data_fetched_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{data_type}}", "legendLink": null, @@ -2187,7 +2187,7 @@ { "expr": "sum by(data_type) (rate(cortex_bucket_store_series_data_touched_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{data_type}}", "legendLink": null, @@ -2276,7 +2276,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_series_get_all_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -2285,7 +2285,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_series_get_all_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -2294,7 +2294,7 @@ { "expr": "sum(rate(cortex_bucket_store_series_get_all_duration_seconds_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_bucket_store_series_get_all_duration_seconds_count{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -2371,7 +2371,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_series_merge_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -2380,7 +2380,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_series_merge_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -2389,7 +2389,7 @@ { "expr": "sum(rate(cortex_bucket_store_series_merge_duration_seconds_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_bucket_store_series_merge_duration_seconds_count{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -2466,7 +2466,7 @@ { "expr": "sum(rate(cortex_bucket_store_series_result_series_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) / sum(rate(cortex_bucket_store_series_result_series_count{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "avg series returned", "legendLink": null, @@ -2555,7 +2555,7 @@ { "expr": "cortex_bucket_store_blocks_loaded{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{pod}}", "legendLink": null, @@ -2635,7 +2635,7 @@ { "expr": "sum(rate(cortex_bucket_store_block_loads_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "successful", "legendLink": null, @@ -2644,7 +2644,7 @@ { "expr": "sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "failed", "legendLink": null, @@ -2724,7 +2724,7 @@ { "expr": "sum(rate(cortex_bucket_store_block_drops_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "successful", "legendLink": null, @@ -2733,7 +2733,7 @@ { "expr": "sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "failed", "legendLink": null, @@ -2822,7 +2822,7 @@ { "expr": "cortex_bucket_store_indexheader_lazy_load_total{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"} - cortex_bucket_store_indexheader_lazy_unload_total{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{pod}}", "legendLink": null, @@ -2899,7 +2899,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -2908,7 +2908,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -2917,7 +2917,7 @@ { "expr": "sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", diff --git a/assets/cortex/dashboards/cortex-reads.json b/assets/cortex/dashboards/cortex-reads.json index 135d0bf..f598e14 100644 --- a/assets/cortex/dashboards/cortex-reads.json +++ b/assets/cortex/dashboards/cortex-reads.json @@ -257,7 +257,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -420,7 +420,7 @@ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "", "legendLink": null, @@ -515,7 +515,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -678,7 +678,7 @@ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "", "legendLink": null, @@ -784,7 +784,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -861,7 +861,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -870,7 +870,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -879,7 +879,7 @@ { "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -976,7 +976,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_cache_request_duration_seconds_count{method=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend|cortex$)\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -1165,7 +1165,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -1328,7 +1328,7 @@ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "", "legendLink": null, @@ -1423,7 +1423,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -1586,7 +1586,7 @@ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval])))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "", "legendLink": null, @@ -1681,7 +1681,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",route=~\"/gatewaypb.StoreGateway/.*\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -1844,7 +1844,7 @@ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\", route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval])))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "", "legendLink": null, @@ -1931,7 +1931,7 @@ { "expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n component=\"store-gateway\",\n name=\"index-cache\",\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{operation}}", "legendLink": null, @@ -2008,7 +2008,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -2017,7 +2017,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -2026,7 +2026,7 @@ { "expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -2104,7 +2104,7 @@ { "expr": "sum by(item_type) (\n rate(\n thanos_store_index_cache_hits_total{\n component=\"store-gateway\",\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"\n }[$__rate_interval]\n )\n)\n/\nsum by(item_type) (\n rate(\n thanos_store_index_cache_requests_total{\n component=\"store-gateway\",\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{item_type}}", "legendLink": null, @@ -2193,7 +2193,7 @@ { "expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{operation}}", "legendLink": null, @@ -2270,7 +2270,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -2279,7 +2279,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -2288,7 +2288,7 @@ { "expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -2365,7 +2365,7 @@ { "expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "items", "legendLink": null, @@ -2454,7 +2454,7 @@ { "expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{operation}}", "legendLink": null, @@ -2531,7 +2531,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -2540,7 +2540,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -2549,7 +2549,7 @@ { "expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -2626,7 +2626,7 @@ { "expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "items", "legendLink": null, @@ -2715,7 +2715,7 @@ { "expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{operation}}", "legendLink": null, @@ -2792,7 +2792,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -2801,7 +2801,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -2810,7 +2810,7 @@ { "expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -2887,7 +2887,7 @@ { "expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier|cortex$)\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "items", "legendLink": null, @@ -2976,7 +2976,7 @@ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{operation}}", "legendLink": null, @@ -3053,7 +3053,7 @@ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{operation}}", "legendLink": null, @@ -3130,7 +3130,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -3139,7 +3139,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -3148,7 +3148,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -3225,7 +3225,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -3234,7 +3234,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -3243,7 +3243,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -3332,7 +3332,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -3341,7 +3341,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -3350,7 +3350,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -3427,7 +3427,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -3436,7 +3436,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -3445,7 +3445,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -3522,7 +3522,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -3531,7 +3531,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -3540,7 +3540,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -3617,7 +3617,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -3626,7 +3626,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -3635,7 +3635,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -3724,7 +3724,7 @@ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{operation}}", "legendLink": null, @@ -3801,7 +3801,7 @@ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{operation}}", "legendLink": null, @@ -3878,7 +3878,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -3887,7 +3887,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -3896,7 +3896,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -3973,7 +3973,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -3982,7 +3982,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -3991,7 +3991,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -4080,7 +4080,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -4089,7 +4089,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -4098,7 +4098,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -4175,7 +4175,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -4184,7 +4184,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -4193,7 +4193,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -4270,7 +4270,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -4279,7 +4279,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -4288,7 +4288,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -4365,7 +4365,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -4374,7 +4374,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -4383,7 +4383,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", diff --git a/assets/cortex/dashboards/cortex-rollout-progress.json b/assets/cortex/dashboards/cortex-rollout-progress.json index 4222b10..50a06cb 100644 --- a/assets/cortex/dashboards/cortex-rollout-progress.json +++ b/assets/cortex/dashboards/cortex-rollout-progress.json @@ -1253,7 +1253,7 @@ { "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}))[1h:])\n)\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "writes", "legendLink": null, @@ -1262,7 +1262,7 @@ { "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}))[1h:])\n)\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "reads", "legendLink": null, diff --git a/assets/cortex/dashboards/cortex-writes.json b/assets/cortex/dashboards/cortex-writes.json index c146ff4..dbf26a0 100644 --- a/assets/cortex/dashboards/cortex-writes.json +++ b/assets/cortex/dashboards/cortex-writes.json @@ -407,7 +407,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -570,7 +570,7 @@ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}[$__rate_interval])))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "", "legendLink": null, @@ -665,7 +665,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -828,7 +828,7 @@ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__rate_interval])))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "", "legendLink": null, @@ -923,7 +923,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -1112,7 +1112,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",route=\"/cortex.Ingester/Push\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -1275,7 +1275,7 @@ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\", route=\"/cortex.Ingester/Push\"}[$__rate_interval])))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "", "legendLink": null, @@ -1370,7 +1370,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -1555,7 +1555,7 @@ { "expr": "sum(rate(cortex_ingester_shipper_uploads_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_ingester_shipper_upload_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "successful", "legendLink": null, @@ -1564,7 +1564,7 @@ { "expr": "sum(rate(cortex_ingester_shipper_upload_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "failed", "legendLink": null, @@ -1642,7 +1642,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -1651,7 +1651,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -1660,7 +1660,7 @@ { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -1753,7 +1753,7 @@ { "expr": "sum(rate(cortex_ingester_tsdb_compactions_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "successful", "legendLink": null, @@ -1762,7 +1762,7 @@ { "expr": "sum(rate(cortex_ingester_tsdb_compactions_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "failed", "legendLink": null, @@ -1840,7 +1840,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -1849,7 +1849,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -1858,7 +1858,7 @@ { "expr": "sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -1951,7 +1951,7 @@ { "expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "successful", "legendLink": null, @@ -1960,7 +1960,7 @@ { "expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "failed", "legendLink": null, @@ -2041,7 +2041,7 @@ { "expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "successful", "legendLink": null, @@ -2050,7 +2050,7 @@ { "expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "failed", "legendLink": null, @@ -2128,7 +2128,7 @@ { "expr": "sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) / sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "avg", "legendLink": null, @@ -2208,7 +2208,7 @@ { "expr": "sum(rate(cortex_ingester_wal_corruptions_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "WAL", "legendLink": null, @@ -2217,7 +2217,7 @@ { "expr": "sum(rate(cortex_ingester_tsdb_mmap_chunk_corruptions_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "mmap-ed chunks", "legendLink": null, diff --git a/assets/cortex/dashboards/ruler.json b/assets/cortex/dashboards/ruler.json index 171582a..64ca830 100644 --- a/assets/cortex/dashboards/ruler.json +++ b/assets/cortex/dashboards/ruler.json @@ -376,7 +376,7 @@ { "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n-\nsum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "success", "legendLink": null, @@ -385,7 +385,7 @@ { "expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "failed", "legendLink": null, @@ -462,7 +462,7 @@ { "expr": "sum (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n /\nsum (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "average", "legendLink": null, @@ -559,7 +559,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -748,7 +748,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -825,7 +825,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -834,7 +834,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -843,7 +843,7 @@ { "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -940,7 +940,7 @@ { "expr": "sum by (status) (label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A", @@ -1017,7 +1017,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -1026,7 +1026,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval])) by (le)) * 1e3", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -1035,7 +1035,7 @@ { "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -1124,7 +1124,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) by (le)) * 1", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -1133,7 +1133,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) by (le)) * 1", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -1142,7 +1142,7 @@ { "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -1219,7 +1219,7 @@ { "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) by (le)) * 1", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A", @@ -1228,7 +1228,7 @@ { "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) by (le)) * 1", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B", @@ -1237,7 +1237,7 @@ { "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Average", "refId": "C", @@ -1314,7 +1314,7 @@ { "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[1m])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[1m]))", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "Failure Rate", "legendLink": null, @@ -1403,7 +1403,7 @@ { "expr": "sum by(user) (rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n> 0\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{ user }}", "legendLink": null, @@ -1480,7 +1480,7 @@ { "expr": "sum by(user) (rate(cortex_prometheus_notifications_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_queue_capacity{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval])) > 0\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{ user }}", "legendLink": null, @@ -1557,7 +1557,7 @@ { "expr": "sum by (user) (increase(cortex_prometheus_notifications_dropped_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval])) > 0\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{ user }}", "legendLink": null, @@ -1646,7 +1646,7 @@ { "expr": "sum by(user) (rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval])) > 0", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{ user }}", "legendLink": null, @@ -1723,7 +1723,7 @@ { "expr": "rate(cortex_prometheus_rule_group_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval])\n /\nrate(cortex_prometheus_rule_group_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval])\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{ user }}", "legendLink": null, @@ -1800,7 +1800,7 @@ { "expr": "sum by(rule_group) (rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval])) > 0", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{ rule_group }}", "legendLink": null, @@ -1889,7 +1889,7 @@ { "expr": "sum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}[$__rate_interval]))\n", "format": "time_series", - "interval": "1m", + "interval": "15s", "intervalFactor": 2, "legendFormat": "{{ user }}", "legendLink": null, diff --git a/assets/kubernetes/alerts.yaml b/assets/kubernetes/alerts.yaml index 3f4fc75..1362b14 100644 --- a/assets/kubernetes/alerts.yaml +++ b/assets/kubernetes/alerts.yaml @@ -54,7 +54,7 @@ groups: expr: | ( kube_deployment_spec_replicas{job="kube-state-metrics"} - != + > kube_deployment_status_replicas_available{job="kube-state-metrics"} ) and ( changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m]) diff --git a/site/content/kubernetes/_index.md b/site/content/kubernetes/_index.md index 661a3a4..4439f0b 100644 --- a/site/content/kubernetes/_index.md +++ b/site/content/kubernetes/_index.md @@ -93,7 +93,7 @@ annotations: expr: | ( kube_deployment_spec_replicas{job="kube-state-metrics"} - != + > kube_deployment_status_replicas_available{job="kube-state-metrics"} ) and ( changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m])