2021-12-28 03:25:10 +00:00
{
"annotations" : {
"list" : [ ]
} ,
"editable" : true ,
"gnetId" : null ,
"graphTooltip" : 0 ,
"hideControls" : false ,
"links" : [
{
"asDropdown" : true ,
"icon" : "external link" ,
"includeVars" : true ,
"keepTime" : true ,
"tags" : [
"loki"
] ,
"targetBlank" : false ,
"title" : "Loki Dashboards" ,
"type" : "dashboards"
}
] ,
"refresh" : "10s" ,
"rows" : [
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : {
"1xx" : "#EAB839" ,
"2xx" : "#7EB26D" ,
"3xx" : "#6ED0E0" ,
"4xx" : "#EF843C" ,
"5xx" : "#E24D42" ,
2024-10-25 03:33:09 +00:00
"OK" : "#7EB26D" ,
"cancel" : "#A9A9A9" ,
2021-12-28 03:25:10 +00:00
"error" : "#E24D42" ,
"success" : "#7EB26D"
} ,
"datasource" : "$datasource" ,
2024-10-25 03:33:09 +00:00
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"lineWidth" : 0 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
}
} ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "short"
} ,
"overrides" : [
{
"matcher" : {
"id" : "byName" ,
"options" : "1xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EAB839" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "2xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "3xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#6ED0E0" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "4xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EF843C" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "5xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "OK"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "cancel"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#A9A9A9" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "error"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "success"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
}
]
} ,
2021-12-28 03:25:10 +00:00
"fill" : 10 ,
"id" : 1 ,
"linewidth" : 0 ,
"links" : [ ] ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
2021-12-28 03:25:10 +00:00
"stack" : true ,
"targets" : [
{
2024-10-25 03:33:09 +00:00
"expr" : "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/query-frontend\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"legendFormat" : "{{status}}" ,
2024-10-25 03:33:09 +00:00
"refId" : "A"
2021-12-28 03:25:10 +00:00
}
] ,
"title" : "QPS" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries"
2021-12-28 03:25:10 +00:00
} ,
{
"datasource" : "$datasource" ,
2024-10-25 03:33:09 +00:00
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
2021-12-28 03:25:10 +00:00
} ,
2024-10-25 03:33:09 +00:00
"id" : 2 ,
2021-12-28 03:25:10 +00:00
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
2021-12-28 03:25:10 +00:00
"targets" : [
{
2024-10-25 03:33:09 +00:00
"expr" : "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
2024-10-25 03:33:09 +00:00
"legendFormat" : "{{ route }} 99th percentile" ,
"refId" : "A"
2021-12-28 03:25:10 +00:00
} ,
{
2024-10-25 03:33:09 +00:00
"expr" : "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
2024-10-25 03:33:09 +00:00
"legendFormat" : "{{ route }} 50th percentile" ,
"refId" : "B"
2021-12-28 03:25:10 +00:00
} ,
{
2024-10-25 03:33:09 +00:00
"expr" : "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route) " ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"legendFormat" : "{{ route }} Average" ,
2024-10-25 03:33:09 +00:00
"refId" : "C"
2021-12-28 03:25:10 +00:00
}
] ,
"title" : "Latency" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries" ,
2021-12-28 03:25:10 +00:00
"yaxes" : [
{
"format" : "ms" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
2023-08-10 03:19:51 +00:00
} ,
{
"datasource" : "$datasource" ,
"fieldConfig" : {
2024-10-25 03:33:09 +00:00
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
2023-08-10 03:19:51 +00:00
} ,
"id" : 3 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
"targets" : [
{
2024-10-25 03:33:09 +00:00
"expr" : "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval])) by (le,pod)) * 1e3" ,
"format" : "time_series" ,
"interval" : "1m" ,
"intervalFactor" : 2 ,
2023-08-10 03:19:51 +00:00
"legendFormat" : "__auto" ,
2024-10-25 03:33:09 +00:00
"refId" : "A" ,
"step" : 10
2023-08-10 03:19:51 +00:00
}
] ,
"title" : "Per Pod Latency (p99)" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries"
2021-12-28 03:25:10 +00:00
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
"title" : "Frontend (query-frontend)" ,
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : {
"1xx" : "#EAB839" ,
"2xx" : "#7EB26D" ,
"3xx" : "#6ED0E0" ,
"4xx" : "#EF843C" ,
"5xx" : "#E24D42" ,
2024-10-25 03:33:09 +00:00
"OK" : "#7EB26D" ,
"cancel" : "#A9A9A9" ,
2021-12-28 03:25:10 +00:00
"error" : "#E24D42" ,
"success" : "#7EB26D"
} ,
"datasource" : "$datasource" ,
2024-10-25 03:33:09 +00:00
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"lineWidth" : 0 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
}
} ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "short"
} ,
"overrides" : [
{
"matcher" : {
"id" : "byName" ,
"options" : "1xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EAB839" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "2xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "3xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#6ED0E0" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "4xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EF843C" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "5xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "OK"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "cancel"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#A9A9A9" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "error"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "success"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
}
]
} ,
2021-12-28 03:25:10 +00:00
"fill" : 10 ,
2023-08-10 03:19:51 +00:00
"id" : 4 ,
2021-12-28 03:25:10 +00:00
"linewidth" : 0 ,
"links" : [ ] ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
2021-12-28 03:25:10 +00:00
"stack" : true ,
"targets" : [
{
2024-10-25 03:33:09 +00:00
"expr" : "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/querier\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"legendFormat" : "{{status}}" ,
2024-10-25 03:33:09 +00:00
"refId" : "A"
2021-12-28 03:25:10 +00:00
}
] ,
"title" : "QPS" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries"
2021-12-28 03:25:10 +00:00
} ,
{
"datasource" : "$datasource" ,
2024-10-25 03:33:09 +00:00
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
2021-12-28 03:25:10 +00:00
} ,
2024-10-25 03:33:09 +00:00
"id" : 5 ,
2021-12-28 03:25:10 +00:00
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
2021-12-28 03:25:10 +00:00
"targets" : [
{
2024-10-25 03:33:09 +00:00
"expr" : "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
2024-10-25 03:33:09 +00:00
"legendFormat" : "{{ route }} 99th percentile" ,
"refId" : "A"
2021-12-28 03:25:10 +00:00
} ,
{
2024-10-25 03:33:09 +00:00
"expr" : "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
2024-10-25 03:33:09 +00:00
"legendFormat" : "{{ route }} 50th percentile" ,
"refId" : "B"
2021-12-28 03:25:10 +00:00
} ,
{
2024-10-25 03:33:09 +00:00
"expr" : "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route) " ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"legendFormat" : "{{ route }} Average" ,
2024-10-25 03:33:09 +00:00
"refId" : "C"
2021-12-28 03:25:10 +00:00
}
] ,
"title" : "Latency" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries" ,
2021-12-28 03:25:10 +00:00
"yaxes" : [
{
"format" : "ms" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
2023-08-10 03:19:51 +00:00
} ,
{
"datasource" : "$datasource" ,
"fieldConfig" : {
2024-10-25 03:33:09 +00:00
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
2023-08-10 03:19:51 +00:00
} ,
"id" : 6 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
"targets" : [
{
2024-10-25 03:33:09 +00:00
"expr" : "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval])) by (le,pod)) * 1e3" ,
"format" : "time_series" ,
"interval" : "1m" ,
"intervalFactor" : 2 ,
2023-08-10 03:19:51 +00:00
"legendFormat" : "__auto" ,
2024-10-25 03:33:09 +00:00
"refId" : "A" ,
"step" : 10
2023-08-10 03:19:51 +00:00
}
] ,
"title" : "Per Pod Latency (p99)" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries"
2021-12-28 03:25:10 +00:00
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
"title" : "Querier" ,
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : {
"1xx" : "#EAB839" ,
"2xx" : "#7EB26D" ,
"3xx" : "#6ED0E0" ,
"4xx" : "#EF843C" ,
"5xx" : "#E24D42" ,
2024-10-25 03:33:09 +00:00
"OK" : "#7EB26D" ,
"cancel" : "#A9A9A9" ,
2021-12-28 03:25:10 +00:00
"error" : "#E24D42" ,
"success" : "#7EB26D"
} ,
"datasource" : "$datasource" ,
2024-10-25 03:33:09 +00:00
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"lineWidth" : 0 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
}
} ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "short"
} ,
"overrides" : [
{
"matcher" : {
"id" : "byName" ,
"options" : "1xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EAB839" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "2xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "3xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#6ED0E0" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "4xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EF843C" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "5xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "OK"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "cancel"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#A9A9A9" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "error"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "success"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
}
]
} ,
2021-12-28 03:25:10 +00:00
"fill" : 10 ,
2023-08-10 03:19:51 +00:00
"id" : 7 ,
2021-12-28 03:25:10 +00:00
"linewidth" : 0 ,
"links" : [ ] ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
2021-12-28 03:25:10 +00:00
"stack" : true ,
"targets" : [
{
2024-10-31 03:33:29 +00:00
"expr" : "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"legendFormat" : "{{status}}" ,
2024-10-25 03:33:09 +00:00
"refId" : "A"
2021-12-28 03:25:10 +00:00
}
] ,
"title" : "QPS" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries"
2021-12-28 03:25:10 +00:00
} ,
{
"datasource" : "$datasource" ,
2024-10-25 03:33:09 +00:00
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
2021-12-28 03:25:10 +00:00
} ,
2024-10-25 03:33:09 +00:00
"id" : 8 ,
2021-12-28 03:25:10 +00:00
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
2021-12-28 03:25:10 +00:00
"targets" : [
{
2024-10-31 03:33:29 +00:00
"expr" : "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
2024-10-25 03:33:09 +00:00
"legendFormat" : "{{ route }} 99th percentile" ,
"refId" : "A"
2021-12-28 03:25:10 +00:00
} ,
{
2024-10-31 03:33:29 +00:00
"expr" : "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
2024-10-25 03:33:09 +00:00
"legendFormat" : "{{ route }} 50th percentile" ,
"refId" : "B"
2021-12-28 03:25:10 +00:00
} ,
{
2024-10-31 03:33:29 +00:00
"expr" : "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) " ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"legendFormat" : "{{ route }} Average" ,
2024-10-25 03:33:09 +00:00
"refId" : "C"
2021-12-28 03:25:10 +00:00
}
] ,
"title" : "Latency" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries" ,
2021-12-28 03:25:10 +00:00
"yaxes" : [
{
"format" : "ms" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
2023-08-10 03:19:51 +00:00
} ,
{
"datasource" : "$datasource" ,
"fieldConfig" : {
2024-10-25 03:33:09 +00:00
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
2023-08-10 03:19:51 +00:00
} ,
"id" : 9 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
"targets" : [
{
2024-10-31 03:33:29 +00:00
"expr" : "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval])) by (le,pod)) * 1e3" ,
2024-10-25 03:33:09 +00:00
"format" : "time_series" ,
"interval" : "1m" ,
"intervalFactor" : 2 ,
2023-08-10 03:19:51 +00:00
"legendFormat" : "__auto" ,
2024-10-25 03:33:09 +00:00
"refId" : "A" ,
"step" : 10
2023-08-10 03:19:51 +00:00
}
] ,
"title" : "Per Pod Latency (p99)" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries"
2021-12-28 03:25:10 +00:00
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
"title" : "Ingester" ,
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : {
"1xx" : "#EAB839" ,
"2xx" : "#7EB26D" ,
"3xx" : "#6ED0E0" ,
"4xx" : "#EF843C" ,
"5xx" : "#E24D42" ,
2024-10-25 03:33:09 +00:00
"OK" : "#7EB26D" ,
"cancel" : "#A9A9A9" ,
2021-12-28 03:25:10 +00:00
"error" : "#E24D42" ,
"success" : "#7EB26D"
} ,
"datasource" : "$datasource" ,
2024-10-25 03:33:09 +00:00
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"lineWidth" : 0 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
}
} ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "short"
} ,
"overrides" : [
{
"matcher" : {
"id" : "byName" ,
"options" : "1xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EAB839" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "2xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "3xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#6ED0E0" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "4xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EF843C" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "5xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "OK"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "cancel"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#A9A9A9" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "error"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "success"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
}
]
} ,
2021-12-28 03:25:10 +00:00
"fill" : 10 ,
2023-08-10 03:19:51 +00:00
"id" : 10 ,
2021-12-28 03:25:10 +00:00
"linewidth" : 0 ,
"links" : [ ] ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
2021-12-28 03:25:10 +00:00
"stack" : true ,
"targets" : [
{
2024-10-31 03:33:29 +00:00
"expr" : "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(partition-ingester-.*|ingester-zone.*)\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"legendFormat" : "{{status}}" ,
2024-10-25 03:33:09 +00:00
"refId" : "A"
2021-12-28 03:25:10 +00:00
}
] ,
"title" : "QPS" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries"
} ,
{
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
2021-12-28 03:25:10 +00:00
} ,
2024-10-25 03:33:09 +00:00
"id" : 11 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
2021-12-28 03:25:10 +00:00
} ,
2024-10-25 03:33:09 +00:00
"span" : 4 ,
"targets" : [
{
2024-10-31 03:33:29 +00:00
"expr" : "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(partition-ingester-.*|ingester-zone.*)\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3" ,
2024-10-25 03:33:09 +00:00
"format" : "time_series" ,
"legendFormat" : "{{ route }} 99th percentile" ,
"refId" : "A"
} ,
{
2024-10-31 03:33:29 +00:00
"expr" : "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(partition-ingester-.*|ingester-zone.*)\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3" ,
2024-10-25 03:33:09 +00:00
"format" : "time_series" ,
"legendFormat" : "{{ route }} 50th percentile" ,
"refId" : "B"
} ,
{
2024-10-31 03:33:29 +00:00
"expr" : "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(partition-ingester-.*|ingester-zone.*)\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(partition-ingester-.*|ingester-zone.*)\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) " ,
2024-10-25 03:33:09 +00:00
"format" : "time_series" ,
"legendFormat" : "{{ route }} Average" ,
"refId" : "C"
}
] ,
"title" : "Latency" ,
"type" : "timeseries" ,
2021-12-28 03:25:10 +00:00
"yaxes" : [
{
2024-10-25 03:33:09 +00:00
"format" : "ms" ,
2021-12-28 03:25:10 +00:00
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"datasource" : "$datasource" ,
2024-10-25 03:33:09 +00:00
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
2021-12-28 03:25:10 +00:00
} ,
2024-10-25 03:33:09 +00:00
"id" : 12 ,
2021-12-28 03:25:10 +00:00
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
2022-10-07 04:13:56 +00:00
"targets" : [
{
2024-10-31 03:33:29 +00:00
"expr" : "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(partition-ingester-.*|ingester-zone.*)\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval])) by (le,pod)) * 1e3" ,
2022-10-07 04:13:56 +00:00
"format" : "time_series" ,
2024-10-25 03:33:09 +00:00
"interval" : "1m" ,
2022-10-07 04:13:56 +00:00
"intervalFactor" : 2 ,
2024-10-25 03:33:09 +00:00
"legendFormat" : "__auto" ,
2022-10-07 04:13:56 +00:00
"refId" : "A" ,
"step" : 10
2024-10-25 03:33:09 +00:00
}
] ,
"title" : "Per Pod Latency (p99)" ,
"type" : "timeseries"
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
"title" : "Ingester - Zone Aware" ,
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : {
"1xx" : "#EAB839" ,
"2xx" : "#7EB26D" ,
"3xx" : "#6ED0E0" ,
"4xx" : "#EF843C" ,
"5xx" : "#E24D42" ,
"OK" : "#7EB26D" ,
"cancel" : "#A9A9A9" ,
"error" : "#E24D42" ,
"success" : "#7EB26D"
} ,
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"lineWidth" : 0 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
}
} ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "short"
2022-10-07 04:13:56 +00:00
} ,
2024-10-25 03:33:09 +00:00
"overrides" : [
{
"matcher" : {
"id" : "byName" ,
"options" : "1xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EAB839" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "2xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "3xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#6ED0E0" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "4xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EF843C" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "5xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "OK"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "cancel"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#A9A9A9" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "error"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "success"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
}
]
} ,
"fill" : 10 ,
"id" : 13 ,
"linewidth" : 0 ,
"links" : [ ] ,
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
"span" : 4 ,
"stack" : true ,
"targets" : [
2022-10-07 04:13:56 +00:00
{
2024-10-25 03:33:09 +00:00
"expr" : "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/index-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n" ,
2022-10-07 04:13:56 +00:00
"format" : "time_series" ,
2024-10-25 03:33:09 +00:00
"legendFormat" : "{{status}}" ,
"refId" : "A"
}
] ,
"title" : "QPS" ,
"type" : "timeseries"
} ,
{
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
2022-10-07 04:13:56 +00:00
} ,
2024-10-25 03:33:09 +00:00
"overrides" : [ ]
} ,
"id" : 14 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
"span" : 4 ,
"targets" : [
2022-10-07 04:13:56 +00:00
{
2024-10-25 03:33:09 +00:00
"expr" : "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/index-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3" ,
"format" : "time_series" ,
"legendFormat" : "{{ route }} 99th percentile" ,
"refId" : "A"
} ,
{
"expr" : "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/index-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3" ,
"format" : "time_series" ,
"legendFormat" : "{{ route }} 50th percentile" ,
"refId" : "B"
} ,
{
"expr" : "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/index-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/index-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) " ,
2022-10-07 04:13:56 +00:00
"format" : "time_series" ,
"legendFormat" : "{{ route }} Average" ,
2024-10-25 03:33:09 +00:00
"refId" : "C"
2022-10-07 04:13:56 +00:00
}
] ,
"title" : "Latency" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries" ,
2022-10-07 04:13:56 +00:00
"yaxes" : [
{
"format" : "ms" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
2023-08-10 03:19:51 +00:00
} ,
{
"datasource" : "$datasource" ,
"fieldConfig" : {
2024-10-25 03:33:09 +00:00
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
2023-08-10 03:19:51 +00:00
} ,
2024-10-25 03:33:09 +00:00
"id" : 15 ,
2023-08-10 03:19:51 +00:00
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
"targets" : [
{
2024-10-25 03:33:09 +00:00
"expr" : "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/index-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval])) by (le,pod)) * 1e3" ,
"format" : "time_series" ,
"interval" : "1m" ,
"intervalFactor" : 2 ,
2023-08-10 03:19:51 +00:00
"legendFormat" : "__auto" ,
2024-10-25 03:33:09 +00:00
"refId" : "A" ,
"step" : 10
2023-08-10 03:19:51 +00:00
}
] ,
"title" : "Per Pod Latency (p99)" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries"
2022-10-07 04:13:56 +00:00
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
2024-10-25 03:33:09 +00:00
"title" : "Index Gateway" ,
2022-10-07 04:13:56 +00:00
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : {
"1xx" : "#EAB839" ,
"2xx" : "#7EB26D" ,
"3xx" : "#6ED0E0" ,
"4xx" : "#EF843C" ,
"5xx" : "#E24D42" ,
2024-10-25 03:33:09 +00:00
"OK" : "#7EB26D" ,
"cancel" : "#A9A9A9" ,
2022-10-07 04:13:56 +00:00
"error" : "#E24D42" ,
"success" : "#7EB26D"
} ,
"datasource" : "$datasource" ,
2024-10-25 03:33:09 +00:00
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"lineWidth" : 0 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
}
} ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "short"
} ,
"overrides" : [
{
"matcher" : {
"id" : "byName" ,
"options" : "1xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EAB839" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "2xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "3xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#6ED0E0" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "4xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EF843C" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "5xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "OK"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "cancel"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#A9A9A9" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "error"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "success"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
}
]
2022-10-07 04:13:56 +00:00
} ,
2024-10-25 03:33:09 +00:00
"fill" : 10 ,
"id" : 16 ,
2022-10-07 04:13:56 +00:00
"linewidth" : 0 ,
"links" : [ ] ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
2022-10-07 04:13:56 +00:00
"stack" : true ,
"targets" : [
{
2024-10-25 03:33:09 +00:00
"expr" : "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/bloom-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n" ,
2022-10-07 04:13:56 +00:00
"format" : "time_series" ,
"legendFormat" : "{{status}}" ,
2024-10-25 03:33:09 +00:00
"refId" : "A"
2022-10-07 04:13:56 +00:00
}
] ,
"title" : "QPS" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries"
} ,
{
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
2022-10-07 04:13:56 +00:00
} ,
2024-10-25 03:33:09 +00:00
"id" : 17 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
2022-10-07 04:13:56 +00:00
} ,
2024-10-25 03:33:09 +00:00
"span" : 4 ,
"targets" : [
{
"expr" : "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3" ,
"format" : "time_series" ,
"legendFormat" : "{{ route }} 99th percentile" ,
"refId" : "A"
} ,
{
"expr" : "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3" ,
"format" : "time_series" ,
"legendFormat" : "{{ route }} 50th percentile" ,
"refId" : "B"
} ,
{
"expr" : "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) " ,
"format" : "time_series" ,
"legendFormat" : "{{ route }} Average" ,
"refId" : "C"
}
] ,
"title" : "Latency" ,
"type" : "timeseries" ,
2022-10-07 04:13:56 +00:00
"yaxes" : [
{
2024-10-25 03:33:09 +00:00
"format" : "ms" ,
2022-10-07 04:13:56 +00:00
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"datasource" : "$datasource" ,
2024-10-25 03:33:09 +00:00
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
2022-10-07 04:13:56 +00:00
} ,
2024-10-25 03:33:09 +00:00
"id" : 18 ,
2022-10-07 04:13:56 +00:00
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
2022-09-22 04:51:52 +00:00
"targets" : [
{
2024-10-25 03:33:09 +00:00
"expr" : "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval])) by (le,pod)) * 1e3" ,
2022-09-22 04:51:52 +00:00
"format" : "time_series" ,
2024-10-25 03:33:09 +00:00
"interval" : "1m" ,
2022-09-22 04:51:52 +00:00
"intervalFactor" : 2 ,
2024-10-25 03:33:09 +00:00
"legendFormat" : "__auto" ,
2022-09-22 04:51:52 +00:00
"refId" : "A" ,
"step" : 10
2024-10-25 03:33:09 +00:00
}
] ,
"title" : "Per Pod Latency (p99)" ,
"type" : "timeseries"
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
"title" : "Bloom Gateway" ,
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : {
"1xx" : "#EAB839" ,
"2xx" : "#7EB26D" ,
"3xx" : "#6ED0E0" ,
"4xx" : "#EF843C" ,
"5xx" : "#E24D42" ,
"OK" : "#7EB26D" ,
"cancel" : "#A9A9A9" ,
"error" : "#E24D42" ,
"success" : "#7EB26D"
} ,
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"lineWidth" : 0 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
}
} ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "short"
} ,
"overrides" : [
{
"matcher" : {
"id" : "byName" ,
"options" : "1xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EAB839" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "2xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "3xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#6ED0E0" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "4xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EF843C" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "5xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "OK"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "cancel"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#A9A9A9" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "error"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "success"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
}
]
} ,
"fill" : 10 ,
"id" : 19 ,
"linewidth" : 0 ,
"links" : [ ] ,
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
"span" : 4 ,
"stack" : true ,
"targets" : [
{
"expr" : "sum by (status) (\n label_replace(label_replace(rate(loki_index_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n" ,
"format" : "time_series" ,
"legendFormat" : "{{status}}" ,
"refId" : "A"
}
] ,
"title" : "QPS" ,
"type" : "timeseries"
} ,
{
"datasource" : "$datasource" ,
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
} ,
"id" : 20 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
"span" : 4 ,
"targets" : [
{
"expr" : "histogram_quantile(0.99, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval])) by (le)) * 1e3" ,
"format" : "time_series" ,
"legendFormat" : "99th Percentile" ,
"refId" : "A"
2022-09-22 04:51:52 +00:00
} ,
{
"expr" : "histogram_quantile(0.50, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval])) by (le)) * 1e3" ,
"format" : "time_series" ,
"legendFormat" : "50th Percentile" ,
2024-10-25 03:33:09 +00:00
"refId" : "B"
2022-09-22 04:51:52 +00:00
} ,
{
"expr" : "sum(rate(loki_index_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval])) * 1e3 / sum(rate(loki_index_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval]))" ,
"format" : "time_series" ,
"legendFormat" : "Average" ,
2024-10-25 03:33:09 +00:00
"refId" : "C"
2022-09-22 04:51:52 +00:00
}
] ,
"title" : "Latency" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries" ,
2022-09-22 04:51:52 +00:00
"yaxes" : [
{
"format" : "ms" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
2023-08-10 03:19:51 +00:00
} ,
{
"datasource" : "$datasource" ,
"fieldConfig" : {
2024-10-25 03:33:09 +00:00
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
2023-08-10 03:19:51 +00:00
} ,
2024-10-25 03:33:09 +00:00
"id" : 21 ,
2023-08-10 03:19:51 +00:00
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
"targets" : [
{
2024-10-25 03:33:09 +00:00
"expr" : "histogram_quantile(0.99, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval])) by (le,pod)) * 1e3" ,
"format" : "time_series" ,
"interval" : "1m" ,
"intervalFactor" : 2 ,
2023-08-10 03:19:51 +00:00
"legendFormat" : "__auto" ,
2024-10-25 03:33:09 +00:00
"refId" : "A" ,
"step" : 10
2023-08-10 03:19:51 +00:00
}
] ,
"title" : "Per Pod Latency (p99)" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries"
2022-09-22 04:51:52 +00:00
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
2024-10-25 03:33:09 +00:00
"title" : "TSBD Index" ,
2022-09-22 04:51:52 +00:00
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : {
"1xx" : "#EAB839" ,
"2xx" : "#7EB26D" ,
"3xx" : "#6ED0E0" ,
"4xx" : "#EF843C" ,
"5xx" : "#E24D42" ,
2024-10-25 03:33:09 +00:00
"OK" : "#7EB26D" ,
"cancel" : "#A9A9A9" ,
2022-09-22 04:51:52 +00:00
"error" : "#E24D42" ,
"success" : "#7EB26D"
} ,
"datasource" : "$datasource" ,
2024-10-25 03:33:09 +00:00
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 100 ,
"lineWidth" : 0 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "normal"
}
} ,
"min" : 0 ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "short"
} ,
"overrides" : [
{
"matcher" : {
"id" : "byName" ,
"options" : "1xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EAB839" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "2xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "3xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#6ED0E0" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "4xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#EF843C" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "5xx"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "OK"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "cancel"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#A9A9A9" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "error"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#E24D42" ,
"mode" : "fixed"
}
}
]
} ,
{
"matcher" : {
"id" : "byName" ,
"options" : "success"
} ,
"properties" : [
{
"id" : "color" ,
"value" : {
"fixedColor" : "#7EB26D" ,
"mode" : "fixed"
}
}
]
}
]
2022-09-22 04:51:52 +00:00
} ,
2024-10-25 03:33:09 +00:00
"fill" : 10 ,
"id" : 22 ,
2022-09-22 04:51:52 +00:00
"linewidth" : 0 ,
"links" : [ ] ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
2022-09-22 04:51:52 +00:00
"stack" : true ,
"targets" : [
{
2024-10-25 03:33:09 +00:00
"expr" : "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n" ,
2022-09-22 04:51:52 +00:00
"format" : "time_series" ,
"legendFormat" : "{{status}}" ,
2024-10-25 03:33:09 +00:00
"refId" : "A"
2022-09-22 04:51:52 +00:00
}
] ,
"title" : "QPS" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries"
2022-09-22 04:51:52 +00:00
} ,
{
"datasource" : "$datasource" ,
2024-10-25 03:33:09 +00:00
"fieldConfig" : {
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
2022-09-22 04:51:52 +00:00
} ,
2024-10-25 03:33:09 +00:00
"id" : 23 ,
2022-09-22 04:51:52 +00:00
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
2021-12-28 03:25:10 +00:00
"targets" : [
{
2022-02-26 03:23:35 +00:00
"expr" : "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"legendFormat" : "99th Percentile" ,
2024-10-25 03:33:09 +00:00
"refId" : "A"
2021-12-28 03:25:10 +00:00
} ,
{
2022-02-26 03:23:35 +00:00
"expr" : "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"legendFormat" : "50th Percentile" ,
2024-10-25 03:33:09 +00:00
"refId" : "B"
2021-12-28 03:25:10 +00:00
} ,
{
2022-02-26 03:23:35 +00:00
"expr" : "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval]))" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"legendFormat" : "Average" ,
2024-10-25 03:33:09 +00:00
"refId" : "C"
2021-12-28 03:25:10 +00:00
}
] ,
"title" : "Latency" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries" ,
2021-12-28 03:25:10 +00:00
"yaxes" : [
{
"format" : "ms" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
2023-08-10 03:19:51 +00:00
} ,
{
"datasource" : "$datasource" ,
"fieldConfig" : {
2024-10-25 03:33:09 +00:00
"defaults" : {
"custom" : {
"drawStyle" : "line" ,
"fillOpacity" : 10 ,
"lineWidth" : 1 ,
"pointSize" : 5 ,
"showPoints" : "never" ,
"spanNulls" : false ,
"stacking" : {
"group" : "A" ,
"mode" : "none"
}
} ,
"thresholds" : {
"mode" : "absolute" ,
"steps" : [ ]
} ,
"unit" : "ms"
} ,
"overrides" : [ ]
2023-08-10 03:19:51 +00:00
} ,
2024-10-25 03:33:09 +00:00
"id" : 24 ,
2023-08-10 03:19:51 +00:00
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
2024-10-25 03:33:09 +00:00
"options" : {
"legend" : {
"showLegend" : true
} ,
"tooltip" : {
"mode" : "single" ,
"sort" : "none"
}
} ,
2023-08-10 03:19:51 +00:00
"span" : 4 ,
"targets" : [
{
2024-10-25 03:33:09 +00:00
"expr" : "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le,pod)) * 1e3" ,
"format" : "time_series" ,
"interval" : "1m" ,
"intervalFactor" : 2 ,
2023-08-10 03:19:51 +00:00
"legendFormat" : "__auto" ,
2024-10-25 03:33:09 +00:00
"refId" : "A" ,
"step" : 10
2023-08-10 03:19:51 +00:00
}
] ,
"title" : "Per Pod Latency (p99)" ,
2024-10-25 03:33:09 +00:00
"type" : "timeseries"
2021-12-28 03:25:10 +00:00
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
2024-10-25 03:33:09 +00:00
"title" : "BoltDB Index" ,
2021-12-28 03:25:10 +00:00
"titleSize" : "h6"
}
] ,
"schemaVersion" : 14 ,
"style" : "dark" ,
"tags" : [
"loki"
] ,
"templating" : {
"list" : [
{
"current" : {
"text" : "default" ,
"value" : "default"
} ,
"hide" : 0 ,
2024-10-25 03:33:09 +00:00
"label" : "Data source" ,
2021-12-28 03:25:10 +00:00
"name" : "datasource" ,
"options" : [ ] ,
"query" : "prometheus" ,
"refresh" : 1 ,
"regex" : "" ,
"type" : "datasource"
} ,
{
"allValue" : null ,
"current" : {
"text" : "prod" ,
"value" : "prod"
} ,
"datasource" : "$datasource" ,
"hide" : 0 ,
"includeAll" : false ,
"label" : "cluster" ,
"multi" : false ,
"name" : "cluster" ,
"options" : [ ] ,
"query" : "label_values(loki_build_info, cluster)" ,
"refresh" : 1 ,
"regex" : "" ,
"sort" : 2 ,
"tagValuesQuery" : "" ,
"tags" : [ ] ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
} ,
{
"allValue" : null ,
"current" : {
"text" : "prod" ,
"value" : "prod"
} ,
"datasource" : "$datasource" ,
"hide" : 0 ,
"includeAll" : false ,
"label" : "namespace" ,
"multi" : false ,
"name" : "namespace" ,
"options" : [ ] ,
"query" : "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)" ,
"refresh" : 1 ,
"regex" : "" ,
"sort" : 2 ,
"tagValuesQuery" : "" ,
"tags" : [ ] ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
}
]
} ,
"time" : {
"from" : "now-1h" ,
"to" : "now"
} ,
"timepicker" : {
"refresh_intervals" : [
"5s" ,
"10s" ,
"30s" ,
"1m" ,
"5m" ,
"15m" ,
"30m" ,
"1h" ,
"2h" ,
"1d"
] ,
"time_options" : [
"5m" ,
"15m" ,
"1h" ,
"6h" ,
"12h" ,
"24h" ,
"2d" ,
"7d" ,
"30d"
]
} ,
"timezone" : "utc" ,
"title" : "Loki / Reads" ,
"uid" : "reads" ,
"version" : 0
}
