{ "annotations": { "list": [ ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "hideControls": false, "links": [ ], "refresh": "10s", "rows": [ { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows rate of requests against /query for the given time.", "fill": 10, "id": 1, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ { "dashboard": "Thanos / Query", "includeVars": true, "keepTime": true, "title": "Thanos / Query", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/1../", "color": "#EAB839" }, { "alias": "/2../", "color": "#37872D" }, { "alias": "/3../", "color": "#E0B400" }, { "alias": "/4../", "color": "#1F60C4" }, { "alias": "/5../", "color": "#C4162A" } ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, handler, code) (rate(http_requests_total{handler=\"query\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{handler}} {{code}}", "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Requests Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "error": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of handled requests against /query.", "fill": 10, "id": 2, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ { "dashboard": "Thanos / Query", "includeVars": true, "keepTime": true, "title": "Thanos / Query", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, code) (rate(http_requests_total{handler=\"query\",code=~\"5..\"}[$interval])) / ignoring (code) group_left() sum by (job) (rate(http_requests_total{handler=\"query\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Requests Errors", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows how long has it taken to handle requests.", "fill": 1, "id": 3, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ { "dashboard": "Thanos / Query", "includeVars": true, "keepTime": true, "title": "Thanos / Query", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{handler=\"query\"}[$interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} P99", "legendLink": null, "step": 10 } ], "thresholds": [ { "colorMode": "warning", "fill": true, "line": true, "op": "gt", "value": 0.5, "yaxis": "left" }, { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Latency 99th Percentile", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Instant Query", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows rate of requests against /query_range for the given time range.", "fill": 10, "id": 4, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ { "dashboard": "Thanos / Query", "includeVars": true, "keepTime": true, "title": "Thanos / Query", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/1../", "color": "#EAB839" }, { "alias": "/2../", "color": "#37872D" }, { "alias": "/3../", "color": "#E0B400" }, { "alias": "/4../", "color": "#1F60C4" }, { "alias": "/5../", "color": "#C4162A" } ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, handler, code) (rate(http_requests_total{handler=\"query_range\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{handler}} {{code}}", "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Requests Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "error": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of handled requests against /query_range.", "fill": 10, "id": 5, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ { "dashboard": "Thanos / Query", "includeVars": true, "keepTime": true, "title": "Thanos / Query", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, code) (rate(http_requests_total{handler=\"query_range\",code=~\"5..\"}[$interval])) / ignoring (code) group_left() sum by (job) (rate(http_requests_total{handler=\"query_range\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Requests Errors", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows how long has it taken to handle requests.", "fill": 1, "id": 6, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ { "dashboard": "Thanos / Query", "includeVars": true, "keepTime": true, "title": "Thanos / Query", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{handler=\"query_range\"}[$interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} P99", "legendLink": null, "step": 10 } ], "thresholds": [ { "colorMode": "warning", "fill": true, "line": true, "op": "gt", "value": 0.5, "yaxis": "left" }, { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Latency 99th Percentile", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Range Query", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows rate of handled Unary gRPC requests from queriers.", "fill": 10, "id": 7, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ { "dashboard": "Thanos / Store", "includeVars": true, "keepTime": true, "title": "Thanos / Store", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/Aborted/", "color": "#EAB839" }, { "alias": "/AlreadyExists/", "color": "#37872D" }, { "alias": "/FailedPrecondition/", "color": "#E0B400" }, { "alias": "/Unimplemented/", "color": "#E0B400" }, { "alias": "/InvalidArgument/", "color": "#1F60C4" }, { "alias": "/NotFound/", "color": "#1F60C4" }, { "alias": "/PermissionDenied/", "color": "#1F60C4" }, { "alias": "/Unauthenticated/", "color": "#1F60C4" }, { "alias": "/Canceled/", "color": "#C4162A" }, { "alias": "/DataLoss/", "color": "#C4162A" }, { "alias": "/DeadlineExceeded/", "color": "#C4162A" }, { "alias": "/Internal/", "color": "#C4162A" }, { "alias": "/OutOfRange/", "color": "#C4162A" }, { "alias": "/ResourceExhausted/", "color": "#C4162A" }, { "alias": "/Unavailable/", "color": "#C4162A" }, { "alias": "/Unknown/", "color": "#C4162A" }, { "alias": "/OK/", "color": "#37872D" }, { "alias": "error", "color": "#C4162A" } ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{grpc_type=\"unary\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "gRPC (Unary) Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "error": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", "fill": 10, "id": 8, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ { "dashboard": "Thanos / Store", "includeVars": true, "keepTime": true, "title": "Thanos / Store", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, grpc_code) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",grpc_type=\"unary\"}[$interval])) / ignoring (grpc_code) group_left() sum by (job) (rate(grpc_server_handled_total{grpc_type=\"unary\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "gRPC (Unary) Errors", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows how long has it taken to handle requests from queriers.", "fill": 1, "id": 9, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ { "dashboard": "Thanos / Store", "includeVars": true, "keepTime": true, "title": "Thanos / Store", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{grpc_type=\"unary\"}[$interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} P99", "legendLink": null, "step": 10 } ], "thresholds": [ { "colorMode": "warning", "fill": true, "line": true, "op": "gt", "value": 0.5, "yaxis": "left" }, { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "gRPC Latency 99th Percentile", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Store", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows rate of handled Unary gRPC requests from queriers.", "fill": 10, "id": 10, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ { "dashboard": "Thanos / Sidecar", "includeVars": true, "keepTime": true, "title": "Thanos / Sidecar", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/Aborted/", "color": "#EAB839" }, { "alias": "/AlreadyExists/", "color": "#37872D" }, { "alias": "/FailedPrecondition/", "color": "#E0B400" }, { "alias": "/Unimplemented/", "color": "#E0B400" }, { "alias": "/InvalidArgument/", "color": "#1F60C4" }, { "alias": "/NotFound/", "color": "#1F60C4" }, { "alias": "/PermissionDenied/", "color": "#1F60C4" }, { "alias": "/Unauthenticated/", "color": "#1F60C4" }, { "alias": "/Canceled/", "color": "#C4162A" }, { "alias": "/DataLoss/", "color": "#C4162A" }, { "alias": "/DeadlineExceeded/", "color": "#C4162A" }, { "alias": "/Internal/", "color": "#C4162A" }, { "alias": "/OutOfRange/", "color": "#C4162A" }, { "alias": "/ResourceExhausted/", "color": "#C4162A" }, { "alias": "/Unavailable/", "color": "#C4162A" }, { "alias": "/Unknown/", "color": "#C4162A" }, { "alias": "/OK/", "color": "#37872D" }, { "alias": "error", "color": "#C4162A" } ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{grpc_type=\"unary\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "gRPC (Unary) Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "error": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", "fill": 10, "id": 11, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ { "dashboard": "Thanos / Sidecar", "includeVars": true, "keepTime": true, "title": "Thanos / Sidecar", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, grpc_code) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",grpc_type=\"unary\"}[$interval])) / ignoring (grpc_code) group_left() sum by (job) (rate(grpc_server_handled_total{grpc_type=\"unary\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "gRPC (Unary) Errors", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", "fill": 1, "id": 12, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ { "dashboard": "Thanos / Sidecar", "includeVars": true, "keepTime": true, "title": "Thanos / Sidecar", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{grpc_type=\"unary\"}[$interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} P99", "legendLink": null, "step": 10 } ], "thresholds": [ { "colorMode": "warning", "fill": true, "line": true, "op": "gt", "value": 0.5, "yaxis": "left" }, { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "gRPC (Unary) Latency 99th Percentile", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Sidecar", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows rate of incoming requests.", "fill": 10, "id": 13, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ { "dashboard": "Thanos / Receive", "includeVars": true, "keepTime": true, "title": "Thanos / Receive", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/1../", "color": "#EAB839" }, { "alias": "/2../", "color": "#37872D" }, { "alias": "/3../", "color": "#E0B400" }, { "alias": "/4../", "color": "#1F60C4" }, { "alias": "/5../", "color": "#C4162A" } ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, handler, code) (rate(http_requests_total{handler=\"receive\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{handler}} {{code}}", "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Incoming Requests Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "error": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of handled incoming requests.", "fill": 10, "id": 14, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ { "dashboard": "Thanos / Receive", "includeVars": true, "keepTime": true, "title": "Thanos / Receive", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, code) (rate(http_requests_total{handler=\"receive\",code=~\"5..\"}[$interval])) / ignoring (code) group_left() sum by (job) (rate(http_requests_total{handler=\"receive\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Incoming Requests Errors", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows how long has it taken to handle incoming requests.", "fill": 1, "id": 15, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ { "dashboard": "Thanos / Receive", "includeVars": true, "keepTime": true, "title": "Thanos / Receive", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{handler=\"receive\"}[$interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} P99", "legendLink": null, "step": 10 } ], "thresholds": [ { "colorMode": "warning", "fill": true, "line": true, "op": "gt", "value": 0.5, "yaxis": "left" }, { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Incoming Requests Latency 99th Percentile", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Receive", "titleSize": "h6" }, { "collapse": true, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows rate of alerts that successfully sent to alert manager.", "fill": 10, "id": 16, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ { "dashboard": "Thanos / Rule", "includeVars": true, "keepTime": true, "title": "Thanos / Rule", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, alertmanager) (rate(thanos_alert_sender_alerts_sent_total{}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{alertmanager}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Alert Sent Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "error": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of sent alerts.", "fill": 10, "id": 17, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ { "dashboard": "Thanos / Rule", "includeVars": true, "keepTime": true, "title": "Thanos / Rule", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job) (rate(thanos_alert_sender_errors_total{}[$interval])) / sum by (job) (rate(thanos_alert_sender_alerts_sent_total{}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "error", "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Alert Sent Errors", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows how long has it taken to send alerts to alert manager.", "fill": 1, "id": 18, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ { "dashboard": "Thanos / Rule", "includeVars": true, "keepTime": true, "title": "Thanos / Rule", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_alert_sender_latency_seconds_bucket{}[$interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} P99", "legendLink": null, "step": 10 } ], "thresholds": [ { "colorMode": "warning", "fill": true, "line": true, "op": "gt", "value": 0.5, "yaxis": "left" }, { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Alert Sent Duration", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Rule", "titleSize": "h6" }, { "collapse": true, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows rate of execution for compactions against blocks that are stored in the bucket.", "fill": 10, "id": 19, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ { "dashboard": "Thanos / Compact", "includeVars": true, "keepTime": true, "title": "Thanos / Compact", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job) (rate(thanos_compact_group_compactions_total{}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "compaction {{job}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Compaction Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "error": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of executed compactions against blocks that are stored in the bucket.", "fill": 10, "id": 20, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ { "dashboard": "Thanos / Compact", "includeVars": true, "keepTime": true, "title": "Thanos / Compact", "type": "dashboard" } ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job) (rate(thanos_compact_group_compactions_failures_total{}[$interval])) / sum by (job) (rate(thanos_compact_group_compactions_total{}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "error", "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Compaction Errors", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Compact", "titleSize": "h6" } ], "schemaVersion": 14, "style": "dark", "tags": [ "thanos-mixin" ], "templating": { "list": [ { "current": { "text": "default", "value": "default" }, "hide": 0, "label": null, "name": "datasource", "options": [ ], "query": "prometheus", "refresh": 1, "regex": "", "type": "datasource" }, { "auto": true, "auto_count": 300, "auto_min": "10s", "current": { "text": "5m", "value": "5m" }, "hide": 0, "label": "interval", "name": "interval", "query": "5m,10m,30m,1h,6h,12h", "refresh": 2, "type": "interval" } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "UTC", "title": "Thanos / Overview", "uid": "0cb8830a6e957978796729870f560cda", "version": 0 }