{ "annotations": { "list": [ ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "hideControls": false, "links": [ ], "refresh": "10s", "rows": [ { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows rate of incoming requests.", "fill": 10, "id": 1, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/1../", "color": "#EAB839" }, { "alias": "/2../", "color": "#37872D" }, { "alias": "/3../", "color": "#E0B400" }, { "alias": "/4../", "color": "#1F60C4" }, { "alias": "/5../", "color": "#C4162A" } ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, handler, code) (rate(http_requests_total{job=~\"$job\", handler=\"receive\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{handler}} {{code}}", "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "error": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of handled incoming requests.", "fill": 10, "id": 2, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, code) (rate(http_requests_total{job=~\"$job\", handler=\"receive\",code=~\"5..\"}[$interval])) / ignoring (code) group_left() sum by (job) (rate(http_requests_total{job=~\"$job\", handler=\"receive\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Errors", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows how long has it taken to handle incoming requests in quantiles.", "fill": 1, "id": 3, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "p99", "color": "#FA6400", "fill": 1, "fillGradient": 1 }, { "alias": "p90", "color": "#E0B400", "fill": 1, "fillGradient": 1 }, { "alias": "p50", "color": "#37872D", "fill": 10, "fillGradient": 0 } ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.50, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"receive\"}[$interval]))) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "p50 {{job}}", "logBase": 10, "max": null, "min": null, "step": 10 }, { "expr": "histogram_quantile(0.90, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"receive\"}[$interval]))) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "p90 {{job}}", "logBase": 10, "max": null, "min": null, "step": 10 }, { "expr": "histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"receive\"}[$interval]))) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "p99 {{job}}", "logBase": 10, "max": null, "min": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Duration", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "WRITE - Incoming Request", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "id": 4, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (tenant, code) (rate(http_requests_total{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{code}} - {{tenant}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Rate of write requests (by tenant and code)", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "id": 5, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (tenant, code) (rate(http_requests_total{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code!~\"2..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{code}} - {{tenant}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Number of errors (by tenant and code)", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "id": 6, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (job, tenant) (rate(http_request_duration_seconds_sum{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\"}[$__rate_interval])) / sum by (job, tenant) (http_request_duration_seconds_count{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{tenant}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Average request duration (by tenant)", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "WRITE - Incoming Request (tenant focus)", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "id": 7, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (job, tenant) (rate(http_request_size_bytes_sum{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code=~\"2..\"}[$__rate_interval])) / sum by (job, tenant) (rate(http_request_size_bytes_count{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code=~\"2..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{tenant}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Average successful HTTP request size (per tenant and code, only 2XX)", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "id": 8, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (job, tenant) (rate(http_request_size_bytes_sum{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code!~\"2..\"}[$__rate_interval])) / sum by (job, tenant) (rate(http_request_size_bytes_count{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code!~\"2..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{tenant}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Average failed HTTP request size (per tenant and code, non 2XX)", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "id": 9, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (job, tenant, method) (http_inflight_requests{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{method}} - {{tenant}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Inflight requests (per tenant and method)", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "HTTP requests (tenant focus)", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "id": 10, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 3, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(thanos_receive_write_timeseries_sum{job=~\"$job\", tenant=~\"$tenant\", code=~\"2..\"}[$__rate_interval])) by (job, tenant) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{tenant}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Rate of series received (per tenant, only 2XX)", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "id": 11, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 3, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(thanos_receive_write_timeseries_sum{job=~\"$job\", tenant=~\"$tenant\", code!~\"2..\"}[$__rate_interval])) by (tenant, code) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{code}} - {{tenant}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Rate of series not written (per tenant and code, non 2XX)", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "id": 12, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 3, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(thanos_receive_write_samples_sum{job=~\"$job\", tenant=~\"$tenant\", code=~\"2..\"}[$__rate_interval])) by (job, tenant) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{tenant}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Rate of samples received (per tenant, only 2XX)", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "id": 13, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 3, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(thanos_receive_write_samples_sum{job=~\"$job\", tenant=~\"$tenant\", code!~\"2..\"}[$__rate_interval])) by (tenant, code) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{code}} - {{tenant}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Rate of samples not written (per tenant and code, non 2XX)", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Series & Samples (tenant focus)", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows rate of replications to other receive nodes.", "fill": 1, "id": 14, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (job) (rate(thanos_receive_replications_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "all {{job}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "error": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of replications to other receive nodes.", "fill": 10, "id": 15, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job) (rate(thanos_receive_replications_total{job=~\"$job\", result=\"error\"}[$interval])) / sum by (job) (rate(thanos_receive_replications_total{job=~\"$job\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "error", "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Errors", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "WRITE - Replication", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows rate of forwarded requests to other receive nodes.", "fill": 1, "id": 16, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (job) (rate(thanos_receive_forward_requests_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "all {{job}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "error": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of forwareded requests to other receive nodes.", "fill": 10, "id": 17, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job) (rate(thanos_receive_forward_requests_total{job=~\"$job\", result=\"error\"}[$interval])) / sum by (job) (rate(thanos_receive_forward_requests_total{job=~\"$job\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "error", "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Errors", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "WRITE - Forward Request", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows rate of handled Unary gRPC requests from queriers.", "fill": 10, "id": 18, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/Aborted/", "color": "#EAB839" }, { "alias": "/AlreadyExists/", "color": "#37872D" }, { "alias": "/FailedPrecondition/", "color": "#E0B400" }, { "alias": "/Unimplemented/", "color": "#E0B400" }, { "alias": "/InvalidArgument/", "color": "#1F60C4" }, { "alias": "/NotFound/", "color": "#1F60C4" }, { "alias": "/PermissionDenied/", "color": "#1F60C4" }, { "alias": "/Unauthenticated/", "color": "#1F60C4" }, { "alias": "/Canceled/", "color": "#C4162A" }, { "alias": "/DataLoss/", "color": "#C4162A" }, { "alias": "/DeadlineExceeded/", "color": "#C4162A" }, { "alias": "/Internal/", "color": "#C4162A" }, { "alias": "/OutOfRange/", "color": "#C4162A" }, { "alias": "/ResourceExhausted/", "color": "#C4162A" }, { "alias": "/Unavailable/", "color": "#C4162A" }, { "alias": "/Unknown/", "color": "#C4162A" }, { "alias": "/OK/", "color": "#37872D" }, { "alias": "error", "color": "#C4162A" } ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "error": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", "fill": 10, "id": 19, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, grpc_code) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval])) / ignoring (grpc_code) group_left() sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Errors", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", "fill": 1, "id": 20, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "p99", "color": "#FA6400", "fill": 1, "fillGradient": 1 }, { "alias": "p90", "color": "#E0B400", "fill": 1, "fillGradient": 1 }, { "alias": "p50", "color": "#37872D", "fill": 10, "fillGradient": 0 } ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "p50 {{job}}", "logBase": 10, "max": null, "min": null, "step": 10 }, { "expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "p90 {{job}}", "logBase": 10, "max": null, "min": null, "step": 10 }, { "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "p99 {{job}}", "logBase": 10, "max": null, "min": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Duration", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "WRITE - gRPC (Unary)", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows rate of handled Unary gRPC requests from queriers.", "fill": 10, "id": 21, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/Aborted/", "color": "#EAB839" }, { "alias": "/AlreadyExists/", "color": "#37872D" }, { "alias": "/FailedPrecondition/", "color": "#E0B400" }, { "alias": "/Unimplemented/", "color": "#E0B400" }, { "alias": "/InvalidArgument/", "color": "#1F60C4" }, { "alias": "/NotFound/", "color": "#1F60C4" }, { "alias": "/PermissionDenied/", "color": "#1F60C4" }, { "alias": "/Unauthenticated/", "color": "#1F60C4" }, { "alias": "/Canceled/", "color": "#C4162A" }, { "alias": "/DataLoss/", "color": "#C4162A" }, { "alias": "/DeadlineExceeded/", "color": "#C4162A" }, { "alias": "/Internal/", "color": "#C4162A" }, { "alias": "/OutOfRange/", "color": "#C4162A" }, { "alias": "/ResourceExhausted/", "color": "#C4162A" }, { "alias": "/Unavailable/", "color": "#C4162A" }, { "alias": "/Unknown/", "color": "#C4162A" }, { "alias": "/OK/", "color": "#37872D" }, { "alias": "error", "color": "#C4162A" } ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "error": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", "fill": 10, "id": 22, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, grpc_code) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval])) / ignoring (grpc_code) group_left() sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Errors", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", "fill": 1, "id": 23, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "p99", "color": "#FA6400", "fill": 1, "fillGradient": 1 }, { "alias": "p90", "color": "#E0B400", "fill": 1, "fillGradient": 1 }, { "alias": "p50", "color": "#37872D", "fill": 10, "fillGradient": 0 } ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "p50 {{job}}", "logBase": 10, "max": null, "min": null, "step": 10 }, { "expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "p90 {{job}}", "logBase": 10, "max": null, "min": null, "step": 10 }, { "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "p99 {{job}}", "logBase": 10, "max": null, "min": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Duration", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "READ - gRPC (Unary)", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows rate of handled Streamed gRPC requests from queriers.", "fill": 10, "id": 24, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/Aborted/", "color": "#EAB839" }, { "alias": "/AlreadyExists/", "color": "#37872D" }, { "alias": "/FailedPrecondition/", "color": "#E0B400" }, { "alias": "/Unimplemented/", "color": "#E0B400" }, { "alias": "/InvalidArgument/", "color": "#1F60C4" }, { "alias": "/NotFound/", "color": "#1F60C4" }, { "alias": "/PermissionDenied/", "color": "#1F60C4" }, { "alias": "/Unauthenticated/", "color": "#1F60C4" }, { "alias": "/Canceled/", "color": "#C4162A" }, { "alias": "/DataLoss/", "color": "#C4162A" }, { "alias": "/DeadlineExceeded/", "color": "#C4162A" }, { "alias": "/Internal/", "color": "#C4162A" }, { "alias": "/OutOfRange/", "color": "#C4162A" }, { "alias": "/ResourceExhausted/", "color": "#C4162A" }, { "alias": "/Unavailable/", "color": "#C4162A" }, { "alias": "/Unknown/", "color": "#C4162A" }, { "alias": "/OK/", "color": "#37872D" }, { "alias": "error", "color": "#C4162A" } ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Rate", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { "error": "#E24D42" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", "fill": 10, "id": 25, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (job, grpc_code) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"server_stream\"}[$interval])) / ignoring (grpc_code) group_left() sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Errors", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", "fill": 1, "id": 26, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "p99", "color": "#FA6400", "fill": 1, "fillGradient": 1 }, { "alias": "p90", "color": "#E0B400", "fill": 1, "fillGradient": 1 }, { "alias": "p50", "color": "#37872D", "fill": 10, "fillGradient": 0 } ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "p50 {{job}}", "logBase": 10, "max": null, "min": null, "step": 10 }, { "expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "p90 {{job}}", "logBase": 10, "max": null, "min": null, "step": 10 }, { "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "p99 {{job}}", "logBase": 10, "max": null, "min": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Duration", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "READ - gRPC (Stream)", "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "description": "Shows the relative time of last successful upload to the object-store bucket.", "fill": 1, "id": 27, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 12, "stack": false, "steppedLine": false, "styles": [ { "alias": "Time", "dateFormat": "YYYY-MM-DD HH:mm:ss", "pattern": "Time", "type": "hidden" }, { "alias": "Uploaded Ago", "colorMode": null, "colors": [ ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "link": false, "linkTargetBlank": false, "linkTooltip": "Drill down", "linkUrl": "", "pattern": "Value", "thresholds": [ ], "type": "number", "unit": "s" }, { "alias": "", "colorMode": null, "colors": [ ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "pattern": "/.*/", "thresholds": [ ], "type": "string", "unit": "short" } ], "targets": [ { "expr": "time() - max by (job, bucket) (thanos_objstore_bucket_last_successful_upload_time{job=~\"$job\"})", "format": "table", "instant": true, "intervalFactor": 2, "legendFormat": "", "refId": "A", "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Successful Upload", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "transform": "table", "type": "table", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Last Updated", "titleSize": "h6" }, { "collapse": true, "height": "250px", "panels": [ { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "id": 28, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "go_memstats_alloc_bytes{job=~\"$job\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "alloc all {{instance}}", "legendLink": null, "step": 10 }, { "expr": "go_memstats_heap_alloc_bytes{job=~\"$job\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "alloc heap {{instance}}", "legendLink": null, "step": 10 }, { "expr": "rate(go_memstats_alloc_bytes_total{job=~\"$job\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "alloc rate all {{instance}}", "legendLink": null, "step": 10 }, { "expr": "rate(go_memstats_heap_alloc_bytes{job=~\"$job\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "alloc rate heap {{instance}}", "legendLink": null, "step": 10 }, { "expr": "go_memstats_stack_inuse_bytes{job=~\"$job\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "inuse heap {{instance}}", "legendLink": null, "step": 10 }, { "expr": "go_memstats_heap_inuse_bytes{job=~\"$job\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "inuse stack {{instance}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Memory Used", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "id": 29, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "go_goroutines{job=~\"$job\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "Goroutines", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": { }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$datasource", "fill": 1, "id": 30, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "go_gc_duration_seconds{job=~\"$job\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{quantile}} {{instance}}", "legendLink": null, "step": 10 } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, "title": "GC Time Quantiles", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [ ] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Resources", "titleSize": "h6" } ], "schemaVersion": 14, "style": "dark", "tags": [ "thanos-mixin" ], "templating": { "list": [ { "current": { "text": "default", "value": "default" }, "hide": 0, "label": null, "name": "datasource", "options": [ ], "query": "prometheus", "refresh": 1, "regex": "", "type": "datasource" }, { "allValue": null, "current": { "text": "all", "value": "$__all" }, "datasource": "$datasource", "hide": 0, "includeAll": true, "label": "tenant", "multi": false, "name": "tenant", "options": [ ], "query": "label_values(http_requests_total{job=~\"$job\", tenant!=\"\"}, tenant)", "refresh": 1, "regex": "", "sort": 2, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false }, { "auto": true, "auto_count": 300, "auto_min": "10s", "current": { "text": "5m", "value": "5m" }, "hide": 0, "label": "interval", "name": "interval", "query": "5m,10m,30m,1h,6h,12h", "refresh": 2, "type": "interval" }, { "allValue": null, "current": { "text": "all", "value": "$__all" }, "datasource": "$datasource", "hide": 0, "includeAll": true, "label": "job", "multi": false, "name": "job", "options": [ ], "query": "label_values(up{job=~\".*thanos-receive.*\"}, job)", "refresh": 1, "regex": "", "sort": 2, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "UTC", "title": "Thanos / Receive", "uid": "916a852b00ccc5ed81056644718fa4fb", "version": 0 }