{
  "annotations": {
     "list": [ ]
  },
  "editable": true,
  "gnetId": null,
  "graphTooltip": 0,
  "hideControls": false,
  "links": [ ],
  "refresh": "10s",
  "rows": [
     {
        "collapse": false,
        "height": "250px",
        "panels": [
           {
              "aliasColors": { },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows rate of requests against /query for the given time.",
              "fill": 10,
              "id": 1,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                 {
                    "alias": "/1../",
                    "color": "#EAB839"
                 },
                 {
                    "alias": "/2../",
                    "color": "#37872D"
                 },
                 {
                    "alias": "/3../",
                    "color": "#E0B400"
                 },
                 {
                    "alias": "/4../",
                    "color": "#1F60C4"
                 },
                 {
                    "alias": "/5../",
                    "color": "#C4162A"
                 }
              ],
              "spaceLength": 10,
              "span": 4,
              "stack": true,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "sum by (job, handler, code) (rate(http_requests_total{job=~\"$job\", handler=\"query\"}[$interval]))",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "{{job}} {{handler}} {{code}}",
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Rate",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           },
           {
              "aliasColors": {
                 "error": "#E24D42"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows ratio of errors compared to the total number of handled requests against /query.",
              "fill": 10,
              "id": 2,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [ ],
              "spaceLength": 10,
              "span": 4,
              "stack": true,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "sum by (job, code) (rate(http_requests_total{job=~\"$job\", handler=\"query\",code=~\"5..\"}[$interval])) / ignoring (code) group_left() sum by (job) (rate(http_requests_total{job=~\"$job\", handler=\"query\"}[$interval]))",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Errors",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "percentunit",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           },
           {
              "aliasColors": { },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows how long has it taken to handle requests in quantiles.",
              "fill": 1,
              "id": 3,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                 {
                    "alias": "p99",
                    "color": "#FA6400",
                    "fill": 1,
                    "fillGradient": 1
                 },
                 {
                    "alias": "p90",
                    "color": "#E0B400",
                    "fill": 1,
                    "fillGradient": 1
                 },
                 {
                    "alias": "p50",
                    "color": "#37872D",
                    "fill": 10,
                    "fillGradient": 0
                 }
              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "histogram_quantile(0.50, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"query\"}[$interval]))) * 1",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "p50 {{job}}",
                    "logBase": 10,
                    "max": null,
                    "min": null,
                    "step": 10
                 },
                 {
                    "expr": "histogram_quantile(0.90, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"query\"}[$interval]))) * 1",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "p90 {{job}}",
                    "logBase": 10,
                    "max": null,
                    "min": null,
                    "step": 10
                 },
                 {
                    "expr": "histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"query\"}[$interval]))) * 1",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "p99 {{job}}",
                    "logBase": 10,
                    "max": null,
                    "min": null,
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Duration",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "s",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           }
        ],
        "repeat": null,
        "repeatIteration": null,
        "repeatRowId": null,
        "showTitle": true,
        "title": "Instant Query API",
        "titleSize": "h6"
     },
     {
        "collapse": false,
        "height": "250px",
        "panels": [
           {
              "aliasColors": { },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows rate of requests against /query_range for the given time range.",
              "fill": 10,
              "id": 4,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                 {
                    "alias": "/1../",
                    "color": "#EAB839"
                 },
                 {
                    "alias": "/2../",
                    "color": "#37872D"
                 },
                 {
                    "alias": "/3../",
                    "color": "#E0B400"
                 },
                 {
                    "alias": "/4../",
                    "color": "#1F60C4"
                 },
                 {
                    "alias": "/5../",
                    "color": "#C4162A"
                 }
              ],
              "spaceLength": 10,
              "span": 4,
              "stack": true,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "sum by (job, handler, code) (rate(http_requests_total{job=~\"$job\", handler=\"query_range\"}[$interval]))",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "{{job}} {{handler}} {{code}}",
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Rate",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           },
           {
              "aliasColors": {
                 "error": "#E24D42"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows ratio of errors compared to the total number of handled requests against /query_range.",
              "fill": 10,
              "id": 5,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [ ],
              "spaceLength": 10,
              "span": 4,
              "stack": true,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "sum by (job, code) (rate(http_requests_total{job=~\"$job\", handler=\"query_range\",code=~\"5..\"}[$interval])) / ignoring (code) group_left() sum by (job) (rate(http_requests_total{job=~\"$job\", handler=\"query_range\"}[$interval]))",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Errors",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "percentunit",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           },
           {
              "aliasColors": { },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows how long has it taken to handle requests in quantiles.",
              "fill": 1,
              "id": 6,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                 {
                    "alias": "p99",
                    "color": "#FA6400",
                    "fill": 1,
                    "fillGradient": 1
                 },
                 {
                    "alias": "p90",
                    "color": "#E0B400",
                    "fill": 1,
                    "fillGradient": 1
                 },
                 {
                    "alias": "p50",
                    "color": "#37872D",
                    "fill": 10,
                    "fillGradient": 0
                 }
              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "histogram_quantile(0.50, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"query_range\"}[$interval]))) * 1",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "p50 {{job}}",
                    "logBase": 10,
                    "max": null,
                    "min": null,
                    "step": 10
                 },
                 {
                    "expr": "histogram_quantile(0.90, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"query_range\"}[$interval]))) * 1",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "p90 {{job}}",
                    "logBase": 10,
                    "max": null,
                    "min": null,
                    "step": 10
                 },
                 {
                    "expr": "histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"query_range\"}[$interval]))) * 1",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "p99 {{job}}",
                    "logBase": 10,
                    "max": null,
                    "min": null,
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Duration",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "s",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           }
        ],
        "repeat": null,
        "repeatIteration": null,
        "repeatRowId": null,
        "showTitle": true,
        "title": "Range Query API",
        "titleSize": "h6"
     },
     {
        "collapse": false,
        "height": "250px",
        "panels": [
           {
              "aliasColors": { },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows rate of handled Unary gRPC requests from other queriers.",
              "fill": 10,
              "id": 7,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                 {
                    "alias": "/Aborted/",
                    "color": "#EAB839"
                 },
                 {
                    "alias": "/AlreadyExists/",
                    "color": "#37872D"
                 },
                 {
                    "alias": "/FailedPrecondition/",
                    "color": "#E0B400"
                 },
                 {
                    "alias": "/Unimplemented/",
                    "color": "#E0B400"
                 },
                 {
                    "alias": "/InvalidArgument/",
                    "color": "#1F60C4"
                 },
                 {
                    "alias": "/NotFound/",
                    "color": "#1F60C4"
                 },
                 {
                    "alias": "/PermissionDenied/",
                    "color": "#1F60C4"
                 },
                 {
                    "alias": "/Unauthenticated/",
                    "color": "#1F60C4"
                 },
                 {
                    "alias": "/Canceled/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/DataLoss/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/DeadlineExceeded/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/Internal/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/OutOfRange/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/ResourceExhausted/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/Unavailable/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/Unknown/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/OK/",
                    "color": "#37872D"
                 },
                 {
                    "alias": "error",
                    "color": "#C4162A"
                 }
              ],
              "spaceLength": 10,
              "span": 4,
              "stack": true,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_client_handled_total{job=~\"$job\", grpc_type=\"unary\"}[$interval]))",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}",
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Rate",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           },
           {
              "aliasColors": {
                 "error": "#E24D42"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows ratio of errors compared to the total number of handled requests from other queriers.",
              "fill": 10,
              "id": 8,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [ ],
              "spaceLength": 10,
              "span": 4,
              "stack": true,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "sum by (job, grpc_code) (rate(grpc_client_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"unary\"}[$interval])) / ignoring (grpc_code) group_left() sum by (job) (rate(grpc_client_handled_total{job=~\"$job\", grpc_type=\"unary\"}[$interval]))",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Errors",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "percentunit",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           },
           {
              "aliasColors": { },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows how long has it taken to handle requests from other queriers, in quantiles.",
              "fill": 1,
              "id": 9,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                 {
                    "alias": "p99",
                    "color": "#FA6400",
                    "fill": 1,
                    "fillGradient": 1
                 },
                 {
                    "alias": "p90",
                    "color": "#E0B400",
                    "fill": 1,
                    "fillGradient": 1
                 },
                 {
                    "alias": "p50",
                    "color": "#37872D",
                    "fill": 10,
                    "fillGradient": 0
                 }
              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_client_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "p50 {{job}}",
                    "logBase": 10,
                    "max": null,
                    "min": null,
                    "step": 10
                 },
                 {
                    "expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_client_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "p90 {{job}}",
                    "logBase": 10,
                    "max": null,
                    "min": null,
                    "step": 10
                 },
                 {
                    "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_client_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "p99 {{job}}",
                    "logBase": 10,
                    "max": null,
                    "min": null,
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Duration",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "s",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           }
        ],
        "repeat": null,
        "repeatIteration": null,
        "repeatRowId": null,
        "showTitle": true,
        "title": "gRPC (Unary)",
        "titleSize": "h6"
     },
     {
        "collapse": false,
        "height": "250px",
        "panels": [
           {
              "aliasColors": { },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows rate of handled Streamed gRPC requests from other queriers.",
              "fill": 10,
              "id": 10,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                 {
                    "alias": "/Aborted/",
                    "color": "#EAB839"
                 },
                 {
                    "alias": "/AlreadyExists/",
                    "color": "#37872D"
                 },
                 {
                    "alias": "/FailedPrecondition/",
                    "color": "#E0B400"
                 },
                 {
                    "alias": "/Unimplemented/",
                    "color": "#E0B400"
                 },
                 {
                    "alias": "/InvalidArgument/",
                    "color": "#1F60C4"
                 },
                 {
                    "alias": "/NotFound/",
                    "color": "#1F60C4"
                 },
                 {
                    "alias": "/PermissionDenied/",
                    "color": "#1F60C4"
                 },
                 {
                    "alias": "/Unauthenticated/",
                    "color": "#1F60C4"
                 },
                 {
                    "alias": "/Canceled/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/DataLoss/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/DeadlineExceeded/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/Internal/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/OutOfRange/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/ResourceExhausted/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/Unavailable/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/Unknown/",
                    "color": "#C4162A"
                 },
                 {
                    "alias": "/OK/",
                    "color": "#37872D"
                 },
                 {
                    "alias": "error",
                    "color": "#C4162A"
                 }
              ],
              "spaceLength": 10,
              "span": 4,
              "stack": true,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_client_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}",
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Rate",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           },
           {
              "aliasColors": {
                 "error": "#E24D42"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows ratio of errors compared to the total number of handled requests from other queriers.",
              "fill": 10,
              "id": 11,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [ ],
              "spaceLength": 10,
              "span": 4,
              "stack": true,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "sum by (job, grpc_code) (rate(grpc_client_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"server_stream\"}[$interval])) / ignoring (grpc_code) group_left() sum by (job) (rate(grpc_client_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Errors",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "percentunit",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           },
           {
              "aliasColors": { },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows how long has it taken to handle requests from other queriers, in quantiles",
              "fill": 1,
              "id": 12,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                 {
                    "alias": "p99",
                    "color": "#FA6400",
                    "fill": 1,
                    "fillGradient": 1
                 },
                 {
                    "alias": "p90",
                    "color": "#E0B400",
                    "fill": 1,
                    "fillGradient": 1
                 },
                 {
                    "alias": "p50",
                    "color": "#37872D",
                    "fill": 10,
                    "fillGradient": 0
                 }
              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_client_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "p50 {{job}}",
                    "logBase": 10,
                    "max": null,
                    "min": null,
                    "step": 10
                 },
                 {
                    "expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_client_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "p90 {{job}}",
                    "logBase": 10,
                    "max": null,
                    "min": null,
                    "step": 10
                 },
                 {
                    "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_client_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "p99 {{job}}",
                    "logBase": 10,
                    "max": null,
                    "min": null,
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Duration",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "s",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           }
        ],
        "repeat": null,
        "repeatIteration": null,
        "repeatRowId": null,
        "showTitle": true,
        "title": "gRPC (Stream)",
        "titleSize": "h6"
     },
     {
        "collapse": false,
        "height": "250px",
        "panels": [
           {
              "aliasColors": { },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows rate of DNS lookups to discover stores.",
              "fill": 1,
              "id": 13,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [ ],
              "spaceLength": 10,
              "span": 6,
              "stack": false,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{job=~\"$job\"}[$interval]))",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "lookups {{job}}",
                    "legendLink": null,
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Rate",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           },
           {
              "aliasColors": {
                 "error": "#E24D42"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows ratio of failures compared to the total number of executed DNS lookups.",
              "fill": 10,
              "id": 14,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [ ],
              "spaceLength": 10,
              "span": 6,
              "stack": true,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "sum by (job) (rate(thanos_query_store_apis_dns_failures_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{job=~\"$job\"}[$interval]))",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "error",
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Errors",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "percentunit",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           }
        ],
        "repeat": null,
        "repeatIteration": null,
        "repeatRowId": null,
        "showTitle": true,
        "title": "DNS",
        "titleSize": "h6"
     },
     {
        "collapse": false,
        "height": "250px",
        "panels": [
           {
              "aliasColors": { },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows available capacity of processing queries in parallel.",
              "fill": 1,
              "id": 15,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [ ],
              "spaceLength": 10,
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "max_over_time(thanos_query_concurrent_gate_queries_max{job=~\"$job\"}[$__rate_interval]) - avg_over_time(thanos_query_concurrent_gate_queries_in_flight{job=~\"$job\"}[$__rate_interval])",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "{{job}} - {{pod}}",
                    "legendLink": null,
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Concurrent Capacity",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           }
        ],
        "repeat": null,
        "repeatIteration": null,
        "repeatRowId": null,
        "showTitle": true,
        "title": "Query Concurrency",
        "titleSize": "h6"
     },
     {
        "collapse": true,
        "height": "250px",
        "panels": [
           {
              "aliasColors": { },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "id": 16,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [ ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "go_memstats_alloc_bytes{job=~\"$job\"}",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "alloc all {{instance}}",
                    "legendLink": null,
                    "step": 10
                 },
                 {
                    "expr": "go_memstats_heap_alloc_bytes{job=~\"$job\"}",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "alloc heap {{instance}}",
                    "legendLink": null,
                    "step": 10
                 },
                 {
                    "expr": "rate(go_memstats_alloc_bytes_total{job=~\"$job\"}[30s])",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "alloc rate all {{instance}}",
                    "legendLink": null,
                    "step": 10
                 },
                 {
                    "expr": "rate(go_memstats_heap_alloc_bytes{job=~\"$job\"}[30s])",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "alloc rate heap {{instance}}",
                    "legendLink": null,
                    "step": 10
                 },
                 {
                    "expr": "go_memstats_stack_inuse_bytes{job=~\"$job\"}",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "inuse heap {{instance}}",
                    "legendLink": null,
                    "step": 10
                 },
                 {
                    "expr": "go_memstats_heap_inuse_bytes{job=~\"$job\"}",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "inuse stack {{instance}}",
                    "legendLink": null,
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Memory Used",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "bytes",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           },
           {
              "aliasColors": { },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "id": 17,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [ ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "go_goroutines{job=~\"$job\"}",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "{{instance}}",
                    "legendLink": null,
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Goroutines",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           },
           {
              "aliasColors": { },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "id": 18,
              "legend": {
                 "avg": false,
                 "current": false,
                 "max": false,
                 "min": false,
                 "show": true,
                 "total": false,
                 "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [ ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [ ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                 {
                    "expr": "go_gc_duration_seconds{job=~\"$job\"}",
                    "format": "time_series",
                    "intervalFactor": 2,
                    "legendFormat": "{{quantile}} {{instance}}",
                    "legendLink": null,
                    "step": 10
                 }
              ],
              "thresholds": [ ],
              "timeFrom": null,
              "timeShift": null,
              "title": "GC Time Quantiles",
              "tooltip": {
                 "shared": false,
                 "sort": 0,
                 "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                 "buckets": null,
                 "mode": "time",
                 "name": null,
                 "show": true,
                 "values": [ ]
              },
              "yaxes": [
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": 0,
                    "show": true
                 },
                 {
                    "format": "short",
                    "label": null,
                    "logBase": 1,
                    "max": null,
                    "min": null,
                    "show": false
                 }
              ]
           }
        ],
        "repeat": null,
        "repeatIteration": null,
        "repeatRowId": null,
        "showTitle": true,
        "title": "Resources",
        "titleSize": "h6"
     }
  ],
  "schemaVersion": 14,
  "style": "dark",
  "tags": [
     "thanos-mixin"
  ],
  "templating": {
     "list": [
        {
           "current": {
              "text": "default",
              "value": "default"
           },
           "hide": 0,
           "label": null,
           "name": "datasource",
           "options": [ ],
           "query": "prometheus",
           "refresh": 1,
           "regex": "",
           "type": "datasource"
        },
        {
           "auto": true,
           "auto_count": 300,
           "auto_min": "10s",
           "current": {
              "text": "5m",
              "value": "5m"
           },
           "hide": 0,
           "label": "interval",
           "name": "interval",
           "query": "5m,10m,30m,1h,6h,12h",
           "refresh": 2,
           "type": "interval"
        },
        {
           "allValue": null,
           "current": {
              "text": "all",
              "value": "$__all"
           },
           "datasource": "$datasource",
           "hide": 0,
           "includeAll": true,
           "label": "job",
           "multi": false,
           "name": "job",
           "options": [ ],
           "query": "label_values(up{job=~\".*thanos-query.*\"}, job)",
           "refresh": 1,
           "regex": "",
           "sort": 2,
           "tagValuesQuery": "",
           "tags": [ ],
           "tagsQuery": "",
           "type": "query",
           "useTags": false
        }
     ]
  },
  "time": {
     "from": "now-1h",
     "to": "now"
  },
  "timepicker": {
     "refresh_intervals": [
        "5s",
        "10s",
        "30s",
        "1m",
        "5m",
        "15m",
        "30m",
        "1h",
        "2h",
        "1d"
     ],
     "time_options": [
        "5m",
        "15m",
        "1h",
        "6h",
        "12h",
        "24h",
        "2d",
        "7d",
        "30d"
     ]
  },
  "timezone": "UTC",
  "title": "Thanos / Query",
  "uid": "af36c91291a603f1d9fbdabdd127ac4a",
  "version": 0
}