mirrors/monitoring-mixins-website
1
0
Fork 0
mirror of https://github.com/monitoring-mixins/website.git synced 2024-12-14 11:37:31 +00:00
Code Activity

assets,site/content: daily assets regeneration

Browse source
This commit is contained in:
github-actions[bot] 2021-08-27 03:23:04 +00:00
parent f75974141b
commit eaed016155
18 changed files with 630 additions and 624 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

116
assets/cortex/dashboards/alertmanager.json
View file

@ -71,7 +71,7 @@
"timeShift": null,
"title": "Total Alerts",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -147,7 +147,7 @@
"timeShift": null,
"title": "Total Silences",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -223,7 +223,7 @@
"timeShift": null,
"title": "Tenants",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -321,7 +321,7 @@
"timeShift": null,
"title": "APS",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -419,7 +419,7 @@
"timeShift": null,
"title": "NPS",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -505,7 +505,7 @@
"timeShift": null,
"title": "NPS by integration",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -568,7 +568,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -577,7 +577,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -586,7 +586,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_alertmanager_notification_latency_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval])) * 1e3 / sum(rate(cortex_alertmanager_notification_latency_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval]))",
"expr": "sum(rate(cortex_alertmanager_notification_latency_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_alertmanager_notification_latency_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -600,7 +600,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -683,7 +683,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_v1_alerts|alertmanager\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_v1_alerts|alertmanager\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -697,7 +697,7 @@
"timeShift": null,
"title": "QPS",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -789,7 +789,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -878,7 +878,7 @@
"timeShift": null,
"title": "Operations / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -955,7 +955,7 @@
"timeShift": null,
"title": "Error rate",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1018,7 +1018,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1027,7 +1027,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1036,7 +1036,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1050,7 +1050,7 @@
"timeShift": null,
"title": "Latency of Op: Attributes",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1113,7 +1113,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1122,7 +1122,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1131,7 +1131,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1145,7 +1145,7 @@
"timeShift": null,
"title": "Latency of Op: Exists",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1220,7 +1220,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1229,7 +1229,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1238,7 +1238,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1252,7 +1252,7 @@
"timeShift": null,
"title": "Latency of Op: Get",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1315,7 +1315,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1324,7 +1324,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1333,7 +1333,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1347,7 +1347,7 @@
"timeShift": null,
"title": "Latency of Op: GetRange",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1410,7 +1410,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1419,7 +1419,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1428,7 +1428,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1442,7 +1442,7 @@
"timeShift": null,
"title": "Latency of Op: Upload",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1505,7 +1505,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1514,7 +1514,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1523,7 +1523,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1537,7 +1537,7 @@
"timeShift": null,
"title": "Latency of Op: Delete",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1626,7 +1626,7 @@
"timeShift": null,
"title": "Per pod Tenants",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1703,7 +1703,7 @@
"timeShift": null,
"title": "Per pod Alerts",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1780,7 +1780,7 @@
"timeShift": null,
"title": "Per pod Silences",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1878,7 +1878,7 @@
"timeShift": null,
"title": "Syncs/sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1955,7 +1955,7 @@
"timeShift": null,
"title": "Syncs/sec (By Reason)",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2032,7 +2032,7 @@
"timeShift": null,
"title": "Ring Check Errors/sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2121,7 +2121,7 @@
"timeShift": null,
"title": "Initial syncs/sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2184,7 +2184,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2193,7 +2193,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2202,7 +2202,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval])) * 1e3 / sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__interval]))",
"expr": "sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2216,7 +2216,7 @@
"timeShift": null,
"title": "Initial sync duration",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2302,7 +2302,7 @@
"timeShift": null,
"title": "Fetch state from other alertmanagers /sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2400,7 +2400,7 @@
"timeShift": null,
"title": "Replicate state to other alertmanagers /sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2486,7 +2486,7 @@
"timeShift": null,
"title": "Merge state from other alertmanagers /sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2572,7 +2572,7 @@
"timeShift": null,
"title": "Persist state to remote storage /sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2621,8 +2621,8 @@
"list": [
{
"current": {
"text": "Prometheus",
"value": "Prometheus"
"text": "default",
"value": "default"
},
"hide": 0,
"label": null,
@ -2634,7 +2634,7 @@
"type": "datasource"
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",
@ -2658,7 +2658,7 @@
"useTags": false
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",

18
assets/cortex/dashboards/cortex-compactor-resources.json
View file

@ -339,7 +339,7 @@
"timeShift": null,
"title": "Receive Bandwidth",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -416,7 +416,7 @@
"timeShift": null,
"title": "Transmit Bandwidth",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -505,7 +505,7 @@
"timeShift": null,
"title": "Disk Writes",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -582,7 +582,7 @@
"timeShift": null,
"title": "Disk Reads",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -659,7 +659,7 @@
"timeShift": null,
"title": "Disk Space Utilization",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -708,8 +708,8 @@
"list": [
{
"current": {
"text": "Prometheus",
"value": "Prometheus"
"text": "default",
"value": "default"
},
"hide": 0,
"label": null,
@ -721,7 +721,7 @@
"type": "datasource"
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",
@ -745,7 +745,7 @@
"useTags": false
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",

92
assets/cortex/dashboards/cortex-compactor.json
View file

@ -95,7 +95,7 @@
"timeShift": null,
"title": "Per-instance runs / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -173,7 +173,7 @@
"timeShift": null,
"title": "Tenants compaction progress",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -263,7 +263,7 @@
"timeShift": null,
"title": "Compacted blocks / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -327,7 +327,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -336,7 +336,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -345,7 +345,7 @@
"step": 10
},
{
"expr": "sum(rate(prometheus_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) * 1e3 / sum(rate(prometheus_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval]))",
"expr": "sum(rate(prometheus_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) * 1e3 / sum(rate(prometheus_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -359,7 +359,7 @@
"timeShift": null,
"title": "Per-block compaction duration",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -448,7 +448,7 @@
"timeShift": null,
"title": "Average blocks / tenant",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -526,7 +526,7 @@
"timeShift": null,
"title": "Tenants with largest number of blocks",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -615,7 +615,7 @@
"timeShift": null,
"title": "Blocks marked for deletion / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -704,7 +704,7 @@
"timeShift": null,
"title": "Blocks deletions / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -805,7 +805,7 @@
"timeShift": null,
"title": "Metadata Syncs / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -868,7 +868,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -877,7 +877,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -886,7 +886,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_compactor_meta_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval])) * 1e3 / sum(rate(cortex_compactor_meta_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__interval]))",
"expr": "sum(rate(cortex_compactor_meta_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_compactor_meta_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -900,7 +900,7 @@
"timeShift": null,
"title": "Metadata Sync Duration",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -989,7 +989,7 @@
"timeShift": null,
"title": "Operations / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1066,7 +1066,7 @@
"timeShift": null,
"title": "Error rate",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1129,7 +1129,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1138,7 +1138,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1147,7 +1147,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1161,7 +1161,7 @@
"timeShift": null,
"title": "Latency of Op: Attributes",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1224,7 +1224,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1233,7 +1233,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1242,7 +1242,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1256,7 +1256,7 @@
"timeShift": null,
"title": "Latency of Op: Exists",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1331,7 +1331,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1340,7 +1340,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1349,7 +1349,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1363,7 +1363,7 @@
"timeShift": null,
"title": "Latency of Op: Get",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1426,7 +1426,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1435,7 +1435,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1444,7 +1444,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1458,7 +1458,7 @@
"timeShift": null,
"title": "Latency of Op: GetRange",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1521,7 +1521,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1530,7 +1530,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1539,7 +1539,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1553,7 +1553,7 @@
"timeShift": null,
"title": "Latency of Op: Upload",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1616,7 +1616,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1625,7 +1625,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1634,7 +1634,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1648,7 +1648,7 @@
"timeShift": null,
"title": "Latency of Op: Delete",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1697,8 +1697,8 @@
"list": [
{
"current": {
"text": "Prometheus",
"value": "Prometheus"
"text": "default",
"value": "default"
},
"hide": 0,
"label": null,
@ -1710,7 +1710,7 @@
"type": "datasource"
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",
@ -1734,7 +1734,7 @@
"useTags": false
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",

12
assets/cortex/dashboards/cortex-config.json
View file

@ -72,7 +72,7 @@
"timeShift": null,
"title": "Startup config file hashes",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -161,7 +161,7 @@
"timeShift": null,
"title": "Runtime config file hashes",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -210,8 +210,8 @@
"list": [
{
"current": {
"text": "Prometheus",
"value": "Prometheus"
"text": "default",
"value": "default"
},
"hide": 0,
"label": null,
@ -223,7 +223,7 @@
"type": "datasource"
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",
@ -247,7 +247,7 @@
"useTags": false
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",

64
assets/cortex/dashboards/cortex-object-store.json
View file

@ -72,7 +72,7 @@
"timeShift": null,
"title": "RPS / component",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -149,7 +149,7 @@
"timeShift": null,
"title": "Error rate / component",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -238,7 +238,7 @@
"timeShift": null,
"title": "RPS / operation",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -315,7 +315,7 @@
"timeShift": null,
"title": "Error rate / operation",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -390,7 +390,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -399,7 +399,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -408,7 +408,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -422,7 +422,7 @@
"timeShift": null,
"title": "Op: Get",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -485,7 +485,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -494,7 +494,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -503,7 +503,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -517,7 +517,7 @@
"timeShift": null,
"title": "Op: GetRange",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -580,7 +580,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -589,7 +589,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -598,7 +598,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -612,7 +612,7 @@
"timeShift": null,
"title": "Op: Exists",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -687,7 +687,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -696,7 +696,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -705,7 +705,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -719,7 +719,7 @@
"timeShift": null,
"title": "Op: Attributes",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -782,7 +782,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -791,7 +791,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -800,7 +800,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -814,7 +814,7 @@
"timeShift": null,
"title": "Op: Upload",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -877,7 +877,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -886,7 +886,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -895,7 +895,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -909,7 +909,7 @@
"timeShift": null,
"title": "Op: Delete",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -958,8 +958,8 @@
"list": [
{
"current": {
"text": "Prometheus",
"value": "Prometheus"
"text": "default",
"value": "default"
},
"hide": 0,
"label": null,
@ -971,7 +971,7 @@
"type": "datasource"
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",
@ -995,7 +995,7 @@
"useTags": false
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",

128
assets/cortex/dashboards/cortex-queries.json
View file

@ -58,7 +58,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -67,7 +67,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -76,7 +76,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_query_frontend_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_query_frontend_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__interval]))",
"expr": "sum(rate(cortex_query_frontend_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_frontend_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -90,7 +90,7 @@
"timeShift": null,
"title": "Queue Duration",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -153,7 +153,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__interval])) by (le)) * 1",
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -162,7 +162,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__interval])) by (le)) * 1",
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -171,7 +171,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_query_frontend_retries_sum{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_query_frontend_retries_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__interval]))",
"expr": "sum(rate(cortex_query_frontend_retries_sum{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) * 1 / sum(rate(cortex_query_frontend_retries_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -185,7 +185,7 @@
"timeShift": null,
"title": "Retries",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -262,7 +262,7 @@
"timeShift": null,
"title": "Queue Length",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -337,7 +337,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -346,7 +346,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -355,7 +355,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__interval]))",
"expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -369,7 +369,7 @@
"timeShift": null,
"title": "Queue Duration",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -446,7 +446,7 @@
"timeShift": null,
"title": "Queue Length",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -535,7 +535,7 @@
"timeShift": null,
"title": "Cache Hit %",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -612,7 +612,7 @@
"timeShift": null,
"title": "Cache misses",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -701,7 +701,7 @@
"timeShift": null,
"title": "Intervals per Query",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -778,7 +778,7 @@
"timeShift": null,
"title": "Sharded Queries %",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -855,7 +855,7 @@
"timeShift": null,
"title": "Sharding factor",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -944,7 +944,7 @@
"timeShift": null,
"title": "Stages",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1021,7 +1021,7 @@
"timeShift": null,
"title": "Chunk cache misses",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1098,7 +1098,7 @@
"timeShift": null,
"title": "Chunk cache corruptions",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1202,7 +1202,7 @@
"timeShift": null,
"title": "Series per Query",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1294,7 +1294,7 @@
"timeShift": null,
"title": "Chunks per Query",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1386,7 +1386,7 @@
"timeShift": null,
"title": "Samples per Query",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1461,7 +1461,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__interval])) by (le)) * 1",
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1470,7 +1470,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__interval])) by (le)) * 1",
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1479,7 +1479,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__interval]))",
"expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1493,7 +1493,7 @@
"timeShift": null,
"title": "Number of store-gateways hit per Query",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1556,7 +1556,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__interval])) by (le)) * 1",
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1565,7 +1565,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__interval])) by (le)) * 1",
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1574,7 +1574,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__interval]))",
"expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1588,7 +1588,7 @@
"timeShift": null,
"title": "Refetches of missing blocks per Query",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1665,7 +1665,7 @@
"timeShift": null,
"title": "Consistency checks failed",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1772,7 +1772,7 @@
"timeShift": null,
"title": "Bucket indexes loaded (per querier)",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1861,7 +1861,7 @@
"timeShift": null,
"title": "Bucket indexes load / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1924,7 +1924,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1933,7 +1933,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1942,7 +1942,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_bucket_index_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_bucket_index_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__interval]))",
"expr": "sum(rate(cortex_bucket_index_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_index_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1956,7 +1956,7 @@
"timeShift": null,
"title": "Bucket indexes load latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2045,7 +2045,7 @@
"timeShift": null,
"title": "Blocks queried / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2122,7 +2122,7 @@
"timeShift": null,
"title": "Data fetched / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2199,7 +2199,7 @@
"timeShift": null,
"title": "Data touched / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2274,7 +2274,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_series_get_all_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_series_get_all_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2283,7 +2283,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_series_get_all_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_series_get_all_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2292,7 +2292,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_bucket_store_series_get_all_duration_seconds_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_bucket_store_series_get_all_duration_seconds_count{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval]))",
"expr": "sum(rate(cortex_bucket_store_series_get_all_duration_seconds_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_series_get_all_duration_seconds_count{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2306,7 +2306,7 @@
"timeShift": null,
"title": "Series fetch duration (per request)",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2369,7 +2369,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_series_merge_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_series_merge_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2378,7 +2378,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_series_merge_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_series_merge_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2387,7 +2387,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_bucket_store_series_merge_duration_seconds_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_bucket_store_series_merge_duration_seconds_count{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval]))",
"expr": "sum(rate(cortex_bucket_store_series_merge_duration_seconds_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_series_merge_duration_seconds_count{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2401,7 +2401,7 @@
"timeShift": null,
"title": "Series merge duration (per request)",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2478,7 +2478,7 @@
"timeShift": null,
"title": "Series returned (per request)",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2567,7 +2567,7 @@
"timeShift": null,
"title": "Blocks currently loaded",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2656,7 +2656,7 @@
"timeShift": null,
"title": "Blocks loaded / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2745,7 +2745,7 @@
"timeShift": null,
"title": "Blocks dropped / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2834,7 +2834,7 @@
"timeShift": null,
"title": "Lazy loaded index-headers",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2897,7 +2897,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2906,7 +2906,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2915,7 +2915,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__interval]))",
"expr": "sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2929,7 +2929,7 @@
"timeShift": null,
"title": "Index-header lazy load duration",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2978,8 +2978,8 @@
"list": [
{
"current": {
"text": "Prometheus",
"value": "Prometheus"
"text": "default",
"value": "default"
},
"hide": 0,
"label": null,
@ -2991,7 +2991,7 @@
"type": "datasource"
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",
@ -3015,7 +3015,7 @@
"useTags": false
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",

212
assets/cortex/dashboards/cortex-reads.json
View file

@ -95,7 +95,7 @@
"timeShift": null,
"title": "Instant queries / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -172,7 +172,7 @@
"timeShift": null,
"title": "Range queries / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -255,7 +255,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -269,7 +269,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -361,7 +361,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -513,7 +513,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -527,7 +527,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -619,7 +619,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -782,7 +782,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -796,7 +796,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -859,7 +859,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -868,7 +868,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -877,7 +877,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__interval]))",
"expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -891,7 +891,7 @@
"timeShift": null,
"title": "Latency (Time in Queue)",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -974,7 +974,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_cache_request_duration_seconds_count{method=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_cache_request_duration_seconds_count{method=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -988,7 +988,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1080,7 +1080,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1163,7 +1163,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1177,7 +1177,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1269,7 +1269,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1421,7 +1421,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1435,7 +1435,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1527,7 +1527,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1679,7 +1679,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",route=~\"/gatewaypb.StoreGateway/.*\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1693,7 +1693,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1785,7 +1785,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1943,7 +1943,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2006,7 +2006,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2015,7 +2015,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2024,7 +2024,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__interval]))",
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2038,7 +2038,7 @@
"timeShift": null,
"title": "Latency (getmulti)",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2116,7 +2116,7 @@
"timeShift": null,
"title": "Hit ratio",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2205,7 +2205,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2268,7 +2268,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2277,7 +2277,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2286,7 +2286,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__interval]))",
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2300,7 +2300,7 @@
"timeShift": null,
"title": "Latency (getmulti)",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2377,7 +2377,7 @@
"timeShift": null,
"title": "Hit ratio",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2466,7 +2466,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2529,7 +2529,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2538,7 +2538,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2547,7 +2547,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__interval]))",
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2561,7 +2561,7 @@
"timeShift": null,
"title": "Latency (getmulti)",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2638,7 +2638,7 @@
"timeShift": null,
"title": "Hit ratio",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2727,7 +2727,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2790,7 +2790,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2799,7 +2799,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2808,7 +2808,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__interval]))",
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2822,7 +2822,7 @@
"timeShift": null,
"title": "Latency (getmulti)",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2899,7 +2899,7 @@
"timeShift": null,
"title": "Hit ratio",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2988,7 +2988,7 @@
"timeShift": null,
"title": "Operations / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -3065,7 +3065,7 @@
"timeShift": null,
"title": "Error rate",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -3128,7 +3128,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3137,7 +3137,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3146,7 +3146,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3160,7 +3160,7 @@
"timeShift": null,
"title": "Latency of Op: Attributes",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -3223,7 +3223,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3232,7 +3232,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3241,7 +3241,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3255,7 +3255,7 @@
"timeShift": null,
"title": "Latency of Op: Exists",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -3330,7 +3330,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3339,7 +3339,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3348,7 +3348,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3362,7 +3362,7 @@
"timeShift": null,
"title": "Latency of Op: Get",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -3425,7 +3425,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3434,7 +3434,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3443,7 +3443,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3457,7 +3457,7 @@
"timeShift": null,
"title": "Latency of Op: GetRange",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -3520,7 +3520,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3529,7 +3529,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3538,7 +3538,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3552,7 +3552,7 @@
"timeShift": null,
"title": "Latency of Op: Upload",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -3615,7 +3615,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3624,7 +3624,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3633,7 +3633,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3647,7 +3647,7 @@
"timeShift": null,
"title": "Latency of Op: Delete",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -3736,7 +3736,7 @@
"timeShift": null,
"title": "Operations / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -3813,7 +3813,7 @@
"timeShift": null,
"title": "Error rate",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -3876,7 +3876,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3885,7 +3885,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3894,7 +3894,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3908,7 +3908,7 @@
"timeShift": null,
"title": "Latency of Op: Attributes",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -3971,7 +3971,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3980,7 +3980,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -3989,7 +3989,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -4003,7 +4003,7 @@
"timeShift": null,
"title": "Latency of Op: Exists",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -4078,7 +4078,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -4087,7 +4087,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -4096,7 +4096,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -4110,7 +4110,7 @@
"timeShift": null,
"title": "Latency of Op: Get",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -4173,7 +4173,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -4182,7 +4182,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -4191,7 +4191,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -4205,7 +4205,7 @@
"timeShift": null,
"title": "Latency of Op: GetRange",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -4268,7 +4268,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -4277,7 +4277,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -4286,7 +4286,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -4300,7 +4300,7 @@
"timeShift": null,
"title": "Latency of Op: Upload",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -4363,7 +4363,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -4372,7 +4372,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -4381,7 +4381,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -4395,7 +4395,7 @@
"timeShift": null,
"title": "Latency of Op: Delete",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -4444,8 +4444,8 @@
"list": [
{
"current": {
"text": "Prometheus",
"value": "Prometheus"
"text": "default",
"value": "default"
},
"hide": 0,
"label": null,
@ -4457,7 +4457,7 @@
"type": "datasource"
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",
@ -4481,7 +4481,7 @@
"useTags": false
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",

26
assets/cortex/dashboards/cortex-rollout-progress.json
View file

@ -119,7 +119,7 @@
"timeShift": null,
"title": "Rollout progress",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -223,7 +223,7 @@
"timeShift": null,
"title": "Writes - 2xx",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -335,7 +335,7 @@
"timeShift": null,
"title": "Writes - 4xx",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -443,7 +443,7 @@
"timeShift": null,
"title": "Writes - 5xx",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -555,7 +555,7 @@
"timeShift": null,
"title": "Writes 99th Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -659,7 +659,7 @@
"timeShift": null,
"title": "Reads - 2xx",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -771,7 +771,7 @@
"timeShift": null,
"title": "Reads - 4xx",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -879,7 +879,7 @@
"timeShift": null,
"title": "Reads - 5xx",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -991,7 +991,7 @@
"timeShift": null,
"title": "Reads 99th Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1119,7 +1119,7 @@
"timeShift": null,
"title": "Unhealthy pods",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1274,7 +1274,7 @@
"timeShift": null,
"title": "Latency vs 24h ago",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1317,8 +1317,8 @@
"list": [
{
"current": {
"text": "Prometheus",
"value": "Prometheus"
"text": "default",
"value": "default"
},
"hide": 0,
"label": null,

16
assets/cortex/dashboards/cortex-scaling.json
View file

@ -96,6 +96,7 @@
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 0,
"link": false,
"linkTargetBlank": false,
"linkTooltip": "Drill down",
"linkUrl": "",
"pattern": "Value",
@ -110,6 +111,7 @@
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": false,
"linkTargetBlank": false,
"linkTooltip": "Drill down",
"linkUrl": "",
"pattern": "__name__",
@ -124,6 +126,7 @@
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": false,
"linkTargetBlank": false,
"linkTooltip": "Drill down",
"linkUrl": "",
"pattern": "cluster",
@ -138,6 +141,7 @@
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": false,
"linkTargetBlank": false,
"linkTooltip": "Drill down",
"linkUrl": "",
"pattern": "deployment",
@ -152,6 +156,7 @@
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": false,
"linkTargetBlank": false,
"linkTooltip": "Drill down",
"linkUrl": "",
"pattern": "namespace",
@ -166,6 +171,7 @@
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": false,
"linkTargetBlank": false,
"linkTooltip": "Drill down",
"linkUrl": "",
"pattern": "reason",
@ -201,7 +207,7 @@
"timeShift": null,
"title": "Workload-based scaling",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -251,8 +257,8 @@
"list": [
{
"current": {
"text": "Prometheus",
"value": "Prometheus"
"text": "default",
"value": "default"
},
"hide": 0,
"label": null,
@ -264,7 +270,7 @@
"type": "datasource"
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",
@ -288,7 +294,7 @@
"useTags": false
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",

4
assets/cortex/dashboards/cortex-slow-queries.json
View file

@ -167,8 +167,8 @@
"list": [
{
"current": {
"text": "Prometheus",
"value": "Prometheus"
"text": "default",
"value": "default"
},
"hide": 0,
"label": null,

74
assets/cortex/dashboards/cortex-writes.json
View file

@ -94,7 +94,7 @@
"timeShift": null,
"title": "Samples / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -170,7 +170,7 @@
"timeShift": null,
"title": "Active Series",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -246,7 +246,7 @@
"timeShift": null,
"title": "Tenants",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -322,7 +322,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -405,7 +405,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -419,7 +419,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -511,7 +511,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -663,7 +663,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -677,7 +677,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -769,7 +769,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -921,7 +921,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -935,7 +935,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1027,7 +1027,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1110,7 +1110,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",route=\"/cortex.Ingester/Push\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",route=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1124,7 +1124,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1216,7 +1216,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1368,7 +1368,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1382,7 +1382,7 @@
"timeShift": null,
"title": "Requests / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1474,7 +1474,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1576,7 +1576,7 @@
"timeShift": null,
"title": "Uploaded blocks / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1640,7 +1640,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1649,7 +1649,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1658,7 +1658,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1672,7 +1672,7 @@
"timeShift": null,
"title": "Upload latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1774,7 +1774,7 @@
"timeShift": null,
"title": "Compactions / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1838,7 +1838,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1847,7 +1847,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1856,7 +1856,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval])) * 1e3 / sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__interval]))",
"expr": "sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1870,7 +1870,7 @@
"timeShift": null,
"title": "Compactions latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1972,7 +1972,7 @@
"timeShift": null,
"title": "WAL truncations / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2062,7 +2062,7 @@
"timeShift": null,
"title": "Checkpoints created / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2140,7 +2140,7 @@
"timeShift": null,
"title": "WAL truncations latency (includes checkpointing)",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2229,7 +2229,7 @@
"timeShift": null,
"title": "Corruptions / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2278,8 +2278,8 @@
"list": [
{
"current": {
"text": "Prometheus",
"value": "Prometheus"
"text": "default",
"value": "default"
},
"hide": 0,
"label": null,
@ -2291,7 +2291,7 @@
"type": "datasource"
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",
@ -2315,7 +2315,7 @@
"useTags": false
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",

136
assets/cortex/dashboards/ruler.json
View file

@ -71,7 +71,7 @@
"timeShift": null,
"title": "Active Configurations",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -147,7 +147,7 @@
"timeShift": null,
"title": "Total Rules",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -223,7 +223,7 @@
"timeShift": null,
"title": "Read from Ingesters - QPS",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -299,7 +299,7 @@
"timeShift": null,
"title": "Write to Ingesters - QPS",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -397,7 +397,7 @@
"timeShift": null,
"title": "EPS",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -474,7 +474,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -557,7 +557,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -571,7 +571,7 @@
"timeShift": null,
"title": "QPS",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -663,7 +663,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -740,7 +740,7 @@
"timeShift": null,
"title": "Per route p99 Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -823,7 +823,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -837,7 +837,7 @@
"timeShift": null,
"title": "QPS",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -900,7 +900,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -909,7 +909,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -918,7 +918,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__interval]))",
"expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -932,7 +932,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1015,7 +1015,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1029,7 +1029,7 @@
"timeShift": null,
"title": "QPS",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1092,7 +1092,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1101,7 +1101,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1110,7 +1110,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__interval]))",
"expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ruler\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1124,7 +1124,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1199,7 +1199,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) by (le)) * 1",
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1208,7 +1208,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) by (le)) * 1",
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1217,7 +1217,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval]))",
"expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1231,7 +1231,7 @@
"timeShift": null,
"title": "Number of store-gateways hit per Query",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1294,7 +1294,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) by (le)) * 1",
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1303,7 +1303,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) by (le)) * 1",
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1312,7 +1312,7 @@
"step": 10
},
{
"expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__interval]))",
"expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -1326,7 +1326,7 @@
"timeShift": null,
"title": "Refetches of missing blocks per Query",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1403,7 +1403,7 @@
"timeShift": null,
"title": "Consistency checks failed",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1492,7 +1492,7 @@
"timeShift": null,
"title": "Delivery Errors",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1569,7 +1569,7 @@
"timeShift": null,
"title": "Queue Length",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1646,7 +1646,7 @@
"timeShift": null,
"title": "Dropped",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1735,7 +1735,7 @@
"timeShift": null,
"title": "Missed Iterations",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1812,7 +1812,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1889,7 +1889,7 @@
"timeShift": null,
"title": "Failures",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -1978,7 +1978,7 @@
"timeShift": null,
"title": "Latency",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2067,7 +2067,7 @@
"timeShift": null,
"title": "Operations / sec",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2144,7 +2144,7 @@
"timeShift": null,
"title": "Error rate",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2207,7 +2207,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2216,7 +2216,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2225,7 +2225,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2239,7 +2239,7 @@
"timeShift": null,
"title": "Latency of Op: Attributes",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2302,7 +2302,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2311,7 +2311,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2320,7 +2320,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2334,7 +2334,7 @@
"timeShift": null,
"title": "Latency of Op: Exists",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2409,7 +2409,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2418,7 +2418,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2427,7 +2427,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2441,7 +2441,7 @@
"timeShift": null,
"title": "Latency of Op: Get",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2504,7 +2504,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2513,7 +2513,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2522,7 +2522,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2536,7 +2536,7 @@
"timeShift": null,
"title": "Latency of Op: GetRange",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2599,7 +2599,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2608,7 +2608,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2617,7 +2617,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2631,7 +2631,7 @@
"timeShift": null,
"title": "Latency of Op: Upload",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2694,7 +2694,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2703,7 +2703,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__interval])) by (le)) * 1e3",
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2712,7 +2712,7 @@
"step": 10
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__interval]))",
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
"intervalFactor": 2,
@ -2726,7 +2726,7 @@
"timeShift": null,
"title": "Latency of Op: Delete",
"tooltip": {
"shared": true,
"shared": false,
"sort": 0,
"value_type": "individual"
},
@ -2775,8 +2775,8 @@
"list": [
{
"current": {
"text": "Prometheus",
"value": "Prometheus"
"text": "default",
"value": "default"
},
"hide": 0,
"label": null,
@ -2788,7 +2788,7 @@
"type": "datasource"
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",
@ -2812,7 +2812,7 @@
"useTags": false
},
{
"allValue": null,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",

20
assets/thanos/dashboards/bucket_replicate.json
View file

@ -47,7 +47,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_replicate_replication_runs_total{result=\"error\", job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_replicate_replication_runs_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_replicate_replication_runs_total{result=\"error\", job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_replicate_replication_runs_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -123,7 +123,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, result) (rate(thanos_replicate_replication_runs_total{result=\"error\", job=\"$job\"}[$interval]))",
"expr": "sum by (job, result) (rate(thanos_replicate_replication_runs_total{result=\"error\", job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{result}}",
@ -219,7 +219,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_replicate_replication_run_duration_seconds_bucket{result=\"success\", job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_replicate_replication_run_duration_seconds_bucket{result=\"success\", job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -229,7 +229,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_replicate_replication_run_duration_seconds_bucket{result=\"success\", job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_replicate_replication_run_duration_seconds_bucket{result=\"success\", job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -239,7 +239,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_replicate_replication_run_duration_seconds_bucket{result=\"success\", job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_replicate_replication_run_duration_seconds_bucket{result=\"success\", job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -329,7 +329,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(blocks_meta_synced{state=\"loaded\", job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(blocks_meta_synced{state=\"loaded\", job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "meta loads",
@ -337,7 +337,7 @@
"step": 10
},
{
"expr": "sum by (job) (rate(blocks_meta_synced{state=\"failed\", job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(blocks_meta_synced{state=\"failed\", job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "partial meta reads",
@ -345,7 +345,7 @@
"step": 10
},
{
"expr": "sum by (job) (rate(thanos_replicate_blocks_already_replicated_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_replicate_blocks_already_replicated_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "already replicated blocks",
@ -353,7 +353,7 @@
"step": 10
},
{
"expr": "sum by (job) (rate(thanos_replicate_blocks_replicated_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_replicate_blocks_replicated_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "replicated blocks",
@ -361,7 +361,7 @@
"step": 10
},
{
"expr": "sum by (job) (rate(thanos_replicate_objects_replicated_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_replicate_objects_replicated_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "replicated objects",

60
assets/thanos/dashboards/compact.json
View file

@ -46,7 +46,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, group) (rate(thanos_compact_group_compactions_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, group) (rate(thanos_compact_group_compactions_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "compaction {{job}} {{group}}",
@ -125,7 +125,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_compact_group_compactions_failures_total{job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_compact_group_compactions_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_compact_group_compactions_failures_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_compact_group_compactions_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -213,7 +213,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, group) (rate(thanos_compact_downsample_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, group) (rate(thanos_compact_downsample_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "downsample {{job}} {{group}}",
@ -292,7 +292,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_compact_downsample_failed_total{job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_compact_downsample_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_compact_downsample_failed_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_compact_downsample_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -380,7 +380,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_compact_garbage_collection_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_compact_garbage_collection_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "garbage collection {{job}}",
@ -459,7 +459,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_compact_garbage_collection_failures_total{job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_compact_garbage_collection_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_compact_garbage_collection_failures_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_compact_garbage_collection_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -554,7 +554,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_compact_garbage_collection_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_compact_garbage_collection_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -564,7 +564,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_compact_garbage_collection_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_compact_garbage_collection_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -574,7 +574,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_compact_garbage_collection_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_compact_garbage_collection_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -665,7 +665,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_compact_blocks_cleaned_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_compact_blocks_cleaned_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "Blocks cleanup {{job}}",
@ -742,7 +742,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_compact_block_cleanup_failures_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_compact_block_cleanup_failures_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "Blocks cleanup failures {{job}}",
@ -819,7 +819,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_compact_blocks_marked_for_deletion_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_compact_blocks_marked_for_deletion_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "Blocks marked {{job}}",
@ -908,7 +908,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_blocks_meta_syncs_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_blocks_meta_syncs_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "sync {{job}}",
@ -987,7 +987,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_blocks_meta_sync_failures_total{job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_blocks_meta_syncs_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_blocks_meta_sync_failures_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_blocks_meta_syncs_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -1082,7 +1082,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_blocks_meta_sync_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_blocks_meta_sync_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -1092,7 +1092,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_blocks_meta_sync_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_blocks_meta_sync_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -1102,7 +1102,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_blocks_meta_sync_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_blocks_meta_sync_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -1193,7 +1193,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, operation) (rate(thanos_objstore_bucket_operations_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{operation}}",
@ -1272,7 +1272,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_objstore_bucket_operations_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -1367,7 +1367,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -1377,7 +1377,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -1387,7 +1387,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -1477,7 +1477,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_memstats_alloc_bytes{job=\"$job\"}",
"expr": "go_memstats_alloc_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc all {{instance}}",
@ -1485,7 +1485,7 @@
"step": 10
},
{
"expr": "go_memstats_heap_alloc_bytes{job=\"$job\"}",
"expr": "go_memstats_heap_alloc_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc heap {{instance}}",
@ -1493,7 +1493,7 @@
"step": 10
},
{
"expr": "rate(go_memstats_alloc_bytes_total{job=\"$job\"}[30s])",
"expr": "rate(go_memstats_alloc_bytes_total{job=~\"$job\"}[30s])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc rate all {{instance}}",
@ -1501,7 +1501,7 @@
"step": 10
},
{
"expr": "rate(go_memstats_heap_alloc_bytes{job=\"$job\"}[30s])",
"expr": "rate(go_memstats_heap_alloc_bytes{job=~\"$job\"}[30s])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc rate heap {{instance}}",
@ -1509,7 +1509,7 @@
"step": 10
},
{
"expr": "go_memstats_stack_inuse_bytes{job=\"$job\"}",
"expr": "go_memstats_stack_inuse_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "inuse heap {{instance}}",
@ -1517,7 +1517,7 @@
"step": 10
},
{
"expr": "go_memstats_heap_inuse_bytes{job=\"$job\"}",
"expr": "go_memstats_heap_inuse_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "inuse stack {{instance}}",
@ -1593,7 +1593,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_goroutines{job=\"$job\"}",
"expr": "go_goroutines{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{instance}}",
@ -1669,7 +1669,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_gc_duration_seconds{job=\"$job\"}",
"expr": "go_gc_duration_seconds{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{quantile}} {{instance}}",

66
assets/thanos/dashboards/receive.json
View file

@ -67,7 +67,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, handler, code) (rate(http_requests_total{job=\"$job\", handler=\"receive\"}[$interval]))",
"expr": "sum by (job, handler, code) (rate(http_requests_total{job=~\"$job\", handler=\"receive\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{handler}} {{code}}",
@ -145,7 +145,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(http_requests_total{job=\"$job\", handler=\"receive\",code=~\"5..\"}[$interval])) / sum by (job) (rate(http_requests_total{job=\"$job\", handler=\"receive\"}[$interval]))",
"expr": "sum by (job) (rate(http_requests_total{job=~\"$job\", handler=\"receive\",code=~\"5..\"}[$interval])) / sum by (job) (rate(http_requests_total{job=~\"$job\", handler=\"receive\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -240,7 +240,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=\"$job\", handler=\"receive\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"receive\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -250,7 +250,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=\"$job\", handler=\"receive\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"receive\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -260,7 +260,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=\"$job\", handler=\"receive\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"receive\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -351,7 +351,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_receive_replications_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_receive_replications_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "all {{job}}",
@ -430,7 +430,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_receive_replications_total{job=\"$job\", result=\"error\"}[$interval])) / sum by (job) (rate(thanos_receive_replications_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_receive_replications_total{job=~\"$job\", result=\"error\"}[$interval])) / sum by (job) (rate(thanos_receive_replications_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -518,7 +518,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_receive_forward_requests_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_receive_forward_requests_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "all {{job}}",
@ -597,7 +597,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_receive_forward_requests_total{job=\"$job\", result=\"error\"}[$interval])) / sum by (job) (rate(thanos_receive_forward_requests_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_receive_forward_requests_total{job=~\"$job\", result=\"error\"}[$interval])) / sum by (job) (rate(thanos_receive_forward_requests_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -758,7 +758,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))",
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}",
@ -836,7 +836,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))",
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -931,7 +931,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -941,7 +941,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -951,7 +951,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -1115,7 +1115,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))",
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}",
@ -1193,7 +1193,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))",
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -1288,7 +1288,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -1298,7 +1298,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -1308,7 +1308,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -1472,7 +1472,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}",
@ -1550,7 +1550,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=\"$job\", grpc_type=\"server_stream\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"server_stream\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -1645,7 +1645,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -1655,7 +1655,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -1665,7 +1665,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -1790,7 +1790,7 @@
],
"targets": [
{
"expr": "time() - max by (job, bucket) (thanos_objstore_bucket_last_successful_upload_time{job=\"$job\"})",
"expr": "time() - max by (job, bucket) (thanos_objstore_bucket_last_successful_upload_time{job=~\"$job\"})",
"format": "table",
"instant": true,
"intervalFactor": 2,
@ -1880,7 +1880,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_memstats_alloc_bytes{job=\"$job\"}",
"expr": "go_memstats_alloc_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc all {{instance}}",
@ -1888,7 +1888,7 @@
"step": 10
},
{
"expr": "go_memstats_heap_alloc_bytes{job=\"$job\"}",
"expr": "go_memstats_heap_alloc_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc heap {{instance}}",
@ -1896,7 +1896,7 @@
"step": 10
},
{
"expr": "rate(go_memstats_alloc_bytes_total{job=\"$job\"}[30s])",
"expr": "rate(go_memstats_alloc_bytes_total{job=~\"$job\"}[30s])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc rate all {{instance}}",
@ -1904,7 +1904,7 @@
"step": 10
},
{
"expr": "rate(go_memstats_heap_alloc_bytes{job=\"$job\"}[30s])",
"expr": "rate(go_memstats_heap_alloc_bytes{job=~\"$job\"}[30s])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc rate heap {{instance}}",
@ -1912,7 +1912,7 @@
"step": 10
},
{
"expr": "go_memstats_stack_inuse_bytes{job=\"$job\"}",
"expr": "go_memstats_stack_inuse_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "inuse heap {{instance}}",
@ -1920,7 +1920,7 @@
"step": 10
},
{
"expr": "go_memstats_heap_inuse_bytes{job=\"$job\"}",
"expr": "go_memstats_heap_inuse_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "inuse stack {{instance}}",
@ -1996,7 +1996,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_goroutines{job=\"$job\"}",
"expr": "go_goroutines{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{instance}}",
@ -2072,7 +2072,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_gc_duration_seconds{job=\"$job\"}",
"expr": "go_gc_duration_seconds{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{quantile}} {{instance}}",

58
assets/thanos/dashboards/rule.json
View file

@ -45,7 +45,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, strategy) (rate(prometheus_rule_evaluations_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, strategy) (rate(prometheus_rule_evaluations_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{ strategy }}",
@ -121,7 +121,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, strategy) (increase(prometheus_rule_group_iterations_missed_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, strategy) (increase(prometheus_rule_group_iterations_missed_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{ strategy }}",
@ -197,7 +197,7 @@
"steppedLine": false,
"targets": [
{
"expr": "(\n max by(job, rule_group) (prometheus_rule_group_last_duration_seconds{job=\"$job\"})\n >\n sum by(job, rule_group) (prometheus_rule_group_interval_seconds{job=\"$job\"})\n)\n",
"expr": "(\n max by(job, rule_group) (prometheus_rule_group_last_duration_seconds{job=~\"$job\"})\n >\n sum by(job, rule_group) (prometheus_rule_group_interval_seconds{job=~\"$job\"})\n)\n",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{ rule_group }}",
@ -286,7 +286,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, alertmanager) (rate(thanos_alert_sender_alerts_dropped_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, alertmanager) (rate(thanos_alert_sender_alerts_dropped_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{alertmanager}}",
@ -363,7 +363,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, alertmanager) (rate(thanos_alert_sender_alerts_sent_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, alertmanager) (rate(thanos_alert_sender_alerts_sent_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{alertmanager}}",
@ -442,7 +442,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_alert_sender_errors_total{job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_alert_sender_alerts_sent_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_alert_sender_errors_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_alert_sender_alerts_sent_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -537,7 +537,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_alert_sender_latency_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_alert_sender_latency_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -547,7 +547,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_alert_sender_latency_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_alert_sender_latency_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -557,7 +557,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_alert_sender_latency_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_alert_sender_latency_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -648,7 +648,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_alert_queue_alerts_dropped_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_alert_queue_alerts_dropped_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}}",
@ -727,7 +727,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_alert_queue_alerts_dropped_total{job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_alert_queue_alerts_pushed_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_alert_queue_alerts_dropped_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_alert_queue_alerts_pushed_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -888,7 +888,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"unary\"}[$interval]))",
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}",
@ -966,7 +966,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=\"$job\", grpc_type=\"unary\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"unary\"}[$interval]))",
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"unary\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -1061,7 +1061,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -1071,7 +1071,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -1081,7 +1081,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -1245,7 +1245,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}",
@ -1323,7 +1323,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=\"$job\", grpc_type=\"server_stream\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"server_stream\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -1418,7 +1418,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -1428,7 +1428,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -1438,7 +1438,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -1528,7 +1528,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_memstats_alloc_bytes{job=\"$job\"}",
"expr": "go_memstats_alloc_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc all {{instance}}",
@ -1536,7 +1536,7 @@
"step": 10
},
{
"expr": "go_memstats_heap_alloc_bytes{job=\"$job\"}",
"expr": "go_memstats_heap_alloc_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc heap {{instance}}",
@ -1544,7 +1544,7 @@
"step": 10
},
{
"expr": "rate(go_memstats_alloc_bytes_total{job=\"$job\"}[30s])",
"expr": "rate(go_memstats_alloc_bytes_total{job=~\"$job\"}[30s])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc rate all {{instance}}",
@ -1552,7 +1552,7 @@
"step": 10
},
{
"expr": "rate(go_memstats_heap_alloc_bytes{job=\"$job\"}[30s])",
"expr": "rate(go_memstats_heap_alloc_bytes{job=~\"$job\"}[30s])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc rate heap {{instance}}",
@ -1560,7 +1560,7 @@
"step": 10
},
{
"expr": "go_memstats_stack_inuse_bytes{job=\"$job\"}",
"expr": "go_memstats_stack_inuse_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "inuse heap {{instance}}",
@ -1568,7 +1568,7 @@
"step": 10
},
{
"expr": "go_memstats_heap_inuse_bytes{job=\"$job\"}",
"expr": "go_memstats_heap_inuse_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "inuse stack {{instance}}",
@ -1644,7 +1644,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_goroutines{job=\"$job\"}",
"expr": "go_goroutines{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{instance}}",
@ -1720,7 +1720,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_gc_duration_seconds{job=\"$job\"}",
"expr": "go_gc_duration_seconds{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{quantile}} {{instance}}",

48
assets/thanos/dashboards/sidecar.json
View file

@ -119,7 +119,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"unary\"}[$interval]))",
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}",
@ -197,7 +197,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=\"$job\", grpc_type=\"unary\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"unary\"}[$interval]))",
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"unary\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -292,7 +292,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -302,7 +302,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -312,7 +312,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -476,7 +476,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}",
@ -553,7 +553,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=\"$job\", grpc_type=\"server_stream\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"server_stream\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -648,7 +648,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -658,7 +658,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -668,7 +668,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -793,7 +793,7 @@
],
"targets": [
{
"expr": "time() - max by (job, bucket) (thanos_objstore_bucket_last_successful_upload_time{job=\"$job\"})",
"expr": "time() - max by (job, bucket) (thanos_objstore_bucket_last_successful_upload_time{job=~\"$job\"})",
"format": "table",
"instant": true,
"intervalFactor": 2,
@ -883,7 +883,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, operation) (rate(thanos_objstore_bucket_operations_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{operation}}",
@ -961,7 +961,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_objstore_bucket_operations_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -1055,7 +1055,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -1065,7 +1065,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -1075,7 +1075,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -1165,7 +1165,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_memstats_alloc_bytes{job=\"$job\"}",
"expr": "go_memstats_alloc_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc all {{instance}}",
@ -1173,7 +1173,7 @@
"step": 10
},
{
"expr": "go_memstats_heap_alloc_bytes{job=\"$job\"}",
"expr": "go_memstats_heap_alloc_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc heap {{instance}}",
@ -1181,7 +1181,7 @@
"step": 10
},
{
"expr": "rate(go_memstats_alloc_bytes_total{job=\"$job\"}[30s])",
"expr": "rate(go_memstats_alloc_bytes_total{job=~\"$job\"}[30s])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc rate all {{instance}}",
@ -1189,7 +1189,7 @@
"step": 10
},
{
"expr": "rate(go_memstats_heap_alloc_bytes{job=\"$job\"}[30s])",
"expr": "rate(go_memstats_heap_alloc_bytes{job=~\"$job\"}[30s])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc rate heap {{instance}}",
@ -1197,7 +1197,7 @@
"step": 10
},
{
"expr": "go_memstats_stack_inuse_bytes{job=\"$job\"}",
"expr": "go_memstats_stack_inuse_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "inuse heap {{instance}}",
@ -1205,7 +1205,7 @@
"step": 10
},
{
"expr": "go_memstats_heap_inuse_bytes{job=\"$job\"}",
"expr": "go_memstats_heap_inuse_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "inuse stack {{instance}}",
@ -1281,7 +1281,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_goroutines{job=\"$job\"}",
"expr": "go_goroutines{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{instance}}",
@ -1357,7 +1357,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_gc_duration_seconds{job=\"$job\"}",
"expr": "go_gc_duration_seconds{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{quantile}} {{instance}}",

104
assets/thanos/dashboards/store.json
View file

@ -119,7 +119,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"unary\"}[$interval]))",
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}",
@ -197,7 +197,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=\"$job\", grpc_type=\"unary\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"unary\"}[$interval]))",
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"unary\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -292,7 +292,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -302,7 +302,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -312,7 +312,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -476,7 +476,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}",
@ -554,7 +554,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=\"$job\", grpc_type=\"server_stream\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"server_stream\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -649,7 +649,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -659,7 +659,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -669,7 +669,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -760,7 +760,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, operation) (rate(thanos_objstore_bucket_operations_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{operation}}",
@ -837,7 +837,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, operation) (rate(thanos_objstore_bucket_operation_failures_total{job=\"$job\"}[$interval])) / sum by (job, operation) (rate(thanos_objstore_bucket_operations_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, operation) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\"}[$interval])) / sum by (job, operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{operation}}",
@ -914,7 +914,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (job, operation, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, operation, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "P99 {{job}}",
@ -922,7 +922,7 @@
"step": 10
},
{
"expr": "sum by (job, operation) (rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=\"$job\"}[$interval])) * 1 / sum by (job, operation) (rate(thanos_objstore_bucket_operation_duration_seconds_count{job=\"$job\"}[$interval]))",
"expr": "sum by (job, operation) (rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\"}[$interval])) * 1 / sum by (job, operation) (rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "mean {{job}}",
@ -930,7 +930,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.50, sum by (job, operation, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, operation, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "P50 {{job}}",
@ -1019,7 +1019,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_bucket_store_block_loads_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_bucket_store_block_loads_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "block loads",
@ -1098,7 +1098,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_bucket_store_block_load_failures_total{job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_bucket_store_block_loads_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_bucket_store_block_load_failures_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_bucket_store_block_loads_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -1174,7 +1174,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, operation) (rate(thanos_bucket_store_block_drops_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, operation) (rate(thanos_bucket_store_block_drops_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "block drops {{job}}",
@ -1253,7 +1253,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job) (rate(thanos_bucket_store_block_drop_failures_total{job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_bucket_store_block_drops_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_bucket_store_block_drop_failures_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_bucket_store_block_drops_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "error",
@ -1341,7 +1341,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, item_type) (rate(thanos_store_index_cache_requests_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, item_type) (rate(thanos_store_index_cache_requests_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{item_type}}",
@ -1418,7 +1418,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, item_type) (rate(thanos_store_index_cache_hits_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, item_type) (rate(thanos_store_index_cache_hits_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{item_type}}",
@ -1495,7 +1495,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, item_type) (rate(thanos_store_index_cache_items_added_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, item_type) (rate(thanos_store_index_cache_items_added_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{item_type}}",
@ -1572,7 +1572,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum by (job, item_type) (rate(thanos_store_index_cache_items_evicted_total{job=\"$job\"}[$interval]))",
"expr": "sum by (job, item_type) (rate(thanos_store_index_cache_items_evicted_total{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}} {{item_type}}",
@ -1661,7 +1661,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_bucket_store_sent_chunk_size_bytes_bucket{job=\"$job\"}[$interval])))",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_bucket_store_sent_chunk_size_bytes_bucket{job=~\"$job\"}[$interval])))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "P99",
@ -1669,7 +1669,7 @@
"step": 10
},
{
"expr": "sum by (job) (rate(thanos_bucket_store_sent_chunk_size_bytes_sum{job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_bucket_store_sent_chunk_size_bytes_count{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_bucket_store_sent_chunk_size_bytes_sum{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_bucket_store_sent_chunk_size_bytes_count{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "mean",
@ -1677,7 +1677,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_bucket_store_sent_chunk_size_bytes_bucket{job=\"$job\"}[$interval])))",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_bucket_store_sent_chunk_size_bytes_bucket{job=~\"$job\"}[$interval])))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "P50",
@ -1765,7 +1765,7 @@
"steppedLine": false,
"targets": [
{
"expr": "thanos_bucket_store_series_blocks_queried{job=\"$job\", quantile=\"0.99\"}",
"expr": "thanos_bucket_store_series_blocks_queried{job=~\"$job\", quantile=\"0.99\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "P99",
@ -1773,7 +1773,7 @@
"step": 10
},
{
"expr": "sum by (job) (rate(thanos_bucket_store_series_blocks_queried_sum{job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_bucket_store_series_blocks_queried_count{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_bucket_store_series_blocks_queried_sum{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_bucket_store_series_blocks_queried_count{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "mean {{job}}",
@ -1781,7 +1781,7 @@
"step": 10
},
{
"expr": "thanos_bucket_store_series_blocks_queried{job=\"$job\", quantile=\"0.50\"}",
"expr": "thanos_bucket_store_series_blocks_queried{job=~\"$job\", quantile=\"0.50\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "P50",
@ -1858,7 +1858,7 @@
"steppedLine": false,
"targets": [
{
"expr": "thanos_bucket_store_series_data_fetched{job=\"$job\", quantile=\"0.99\"}",
"expr": "thanos_bucket_store_series_data_fetched{job=~\"$job\", quantile=\"0.99\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "P99",
@ -1866,7 +1866,7 @@
"step": 10
},
{
"expr": "sum by (job) (rate(thanos_bucket_store_series_data_fetched_sum{job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_bucket_store_series_data_fetched_count{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_bucket_store_series_data_fetched_sum{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_bucket_store_series_data_fetched_count{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "mean {{job}}",
@ -1874,7 +1874,7 @@
"step": 10
},
{
"expr": "thanos_bucket_store_series_data_fetched{job=\"$job\", quantile=\"0.50\"}",
"expr": "thanos_bucket_store_series_data_fetched{job=~\"$job\", quantile=\"0.50\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "P50",
@ -1950,7 +1950,7 @@
"steppedLine": false,
"targets": [
{
"expr": "thanos_bucket_store_series_result_series{job=\"$job\",quantile=\"0.99\"}",
"expr": "thanos_bucket_store_series_result_series{job=~\"$job\",quantile=\"0.99\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "P99",
@ -1958,7 +1958,7 @@
"step": 10
},
{
"expr": "sum by (job) (rate(thanos_bucket_store_series_result_series_sum{job=\"$job\"}[$interval])) / sum by (job) (rate(thanos_bucket_store_series_result_series_count{job=\"$job\"}[$interval]))",
"expr": "sum by (job) (rate(thanos_bucket_store_series_result_series_sum{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_bucket_store_series_result_series_count{job=~\"$job\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "mean {{job}}",
@ -1966,7 +1966,7 @@
"step": 10
},
{
"expr": "thanos_bucket_store_series_result_series{job=\"$job\",quantile=\"0.50\"}",
"expr": "thanos_bucket_store_series_result_series{job=~\"$job\",quantile=\"0.50\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "P50",
@ -2074,7 +2074,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_bucket_store_series_get_all_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_bucket_store_series_get_all_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -2084,7 +2084,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_bucket_store_series_get_all_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_bucket_store_series_get_all_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -2094,7 +2094,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_bucket_store_series_get_all_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_bucket_store_series_get_all_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -2192,7 +2192,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_bucket_store_series_merge_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_bucket_store_series_merge_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -2202,7 +2202,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_bucket_store_series_merge_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_bucket_store_series_merge_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -2212,7 +2212,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_bucket_store_series_merge_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_bucket_store_series_merge_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -2310,7 +2310,7 @@
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_bucket_store_series_gate_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_bucket_store_series_gate_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p50 {{job}}",
@ -2320,7 +2320,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_bucket_store_series_gate_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_bucket_store_series_gate_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p90 {{job}}",
@ -2330,7 +2330,7 @@
"step": 10
},
{
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_bucket_store_series_gate_duration_seconds_bucket{job=\"$job\"}[$interval]))) * 1",
"expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_bucket_store_series_gate_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "p99 {{job}}",
@ -2420,7 +2420,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_memstats_alloc_bytes{job=\"$job\"}",
"expr": "go_memstats_alloc_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc all {{instance}}",
@ -2428,7 +2428,7 @@
"step": 10
},
{
"expr": "go_memstats_heap_alloc_bytes{job=\"$job\"}",
"expr": "go_memstats_heap_alloc_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc heap {{instance}}",
@ -2436,7 +2436,7 @@
"step": 10
},
{
"expr": "rate(go_memstats_alloc_bytes_total{job=\"$job\"}[30s])",
"expr": "rate(go_memstats_alloc_bytes_total{job=~\"$job\"}[30s])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc rate all {{instance}}",
@ -2444,7 +2444,7 @@
"step": 10
},
{
"expr": "rate(go_memstats_heap_alloc_bytes{job=\"$job\"}[30s])",
"expr": "rate(go_memstats_heap_alloc_bytes{job=~\"$job\"}[30s])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "alloc rate heap {{instance}}",
@ -2452,7 +2452,7 @@
"step": 10
},
{
"expr": "go_memstats_stack_inuse_bytes{job=\"$job\"}",
"expr": "go_memstats_stack_inuse_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "inuse heap {{instance}}",
@ -2460,7 +2460,7 @@
"step": 10
},
{
"expr": "go_memstats_heap_inuse_bytes{job=\"$job\"}",
"expr": "go_memstats_heap_inuse_bytes{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "inuse stack {{instance}}",
@ -2536,7 +2536,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_goroutines{job=\"$job\"}",
"expr": "go_goroutines{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{instance}}",
@ -2612,7 +2612,7 @@
"steppedLine": false,
"targets": [
{
"expr": "go_gc_duration_seconds{job=\"$job\"}",
"expr": "go_gc_duration_seconds{job=~\"$job\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{quantile}} {{instance}}",