monitoring-mixins-website/assets/loki/dashboards/loki-reads-resources.json

{
   "annotations": {
      "list": [ ]
   },
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
   "hideControls": false,
   "links": [
      {
         "asDropdown": true,
         "icon": "external link",
         "includeVars": true,
         "keepTime": true,
         "tags": [
            "loki"
         ],
         "targetBlank": false,
         "title": "Loki Dashboards",
         "type": "dashboards"
      }
   ],
   "refresh": "10s",
   "rows": [
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "request"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#FFC000",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     },
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "limit"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#E02F44",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     }
                  ]
               },
               "id": 1,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"}[$__rate_interval]))",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  },
                  {
                     "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\", resource=\"cpu\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "request",
                     "legendLink": null
                  },
                  {
                     "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"})",
                     "format": "time_series",
                     "legendFormat": "limit",
                     "legendLink": null
                  }
               ],
               "title": "CPU",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "bytes"
                  },
                  "overrides": [
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "request"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#FFC000",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     },
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "limit"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#E02F44",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     }
                  ]
               },
               "id": 2,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  },
                  {
                     "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\", resource=\"memory\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "request",
                     "legendLink": null
                  },
                  {
                     "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "limit",
                     "legendLink": null
                  }
               ],
               "title": "Memory (workingset)",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "bytes"
                  },
                  "overrides": [ ]
               },
               "id": 3,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  }
               ],
               "title": "Memory (go heap inuse)",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "Query Frontend",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "request"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#FFC000",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     },
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "limit"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#E02F44",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     }
                  ]
               },
               "id": 4,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"}[$__rate_interval]))",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  },
                  {
                     "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\", resource=\"cpu\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "request",
                     "legendLink": null
                  },
                  {
                     "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"})",
                     "format": "time_series",
                     "legendFormat": "limit",
                     "legendLink": null
                  }
               ],
               "title": "CPU",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "bytes"
                  },
                  "overrides": [
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "request"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#FFC000",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     },
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "limit"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#E02F44",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     }
                  ]
               },
               "id": 5,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  },
                  {
                     "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\", resource=\"memory\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "request",
                     "legendLink": null
                  },
                  {
                     "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "limit",
                     "legendLink": null
                  }
               ],
               "title": "Memory (workingset)",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "bytes"
                  },
                  "overrides": [ ]
               },
               "id": 6,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  }
               ],
               "title": "Memory (go heap inuse)",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "Query Scheduler",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "request"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#FFC000",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     },
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "limit"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#E02F44",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     }
                  ]
               },
               "id": 7,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"}[$__rate_interval]))",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  },
                  {
                     "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\", resource=\"cpu\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "request",
                     "legendLink": null
                  },
                  {
                     "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"})",
                     "format": "time_series",
                     "legendFormat": "limit",
                     "legendLink": null
                  }
               ],
               "title": "CPU",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "bytes"
                  },
                  "overrides": [
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "request"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#FFC000",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     },
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "limit"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#E02F44",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     }
                  ]
               },
               "id": 8,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  },
                  {
                     "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\", resource=\"memory\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "request",
                     "legendLink": null
                  },
                  {
                     "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "limit",
                     "legendLink": null
                  }
               ],
               "title": "Memory (workingset)",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "bytes"
                  },
                  "overrides": [ ]
               },
               "id": 9,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/querier\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  }
               ],
               "title": "Memory (go heap inuse)",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 100,
                        "lineWidth": 0,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "normal"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "Bps"
                  },
                  "overrides": [ ]
               },
               "id": 10,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "sum by(instance, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n",
                     "format": "time_series",
                     "legendFormat": "{{pod}} - {{device}}",
                     "legendLink": null
                  }
               ],
               "title": "Disk Writes",
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 100,
                        "lineWidth": 0,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "normal"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "Bps"
                  },
                  "overrides": [ ]
               },
               "id": 11,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "sum by(instance,device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n",
                     "format": "time_series",
                     "legendFormat": "{{pod}} - {{device}}",
                     "legendLink": null
                  }
               ],
               "title": "Disk Reads",
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "percentunit"
                  },
                  "overrides": [ ]
               },
               "id": 12,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*querier.*\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*querier.*\"})",
                     "format": "time_series",
                     "legendFormat": "{{persistentvolumeclaim}}",
                     "legendLink": null
                  }
               ],
               "title": "Disk Space Utilization",
               "type": "timeseries"
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "Querier",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "request"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#FFC000",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     },
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "limit"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#E02F44",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     }
                  ]
               },
               "id": 13,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"}[$__rate_interval]))",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  },
                  {
                     "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\", resource=\"cpu\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "request",
                     "legendLink": null
                  },
                  {
                     "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"})",
                     "format": "time_series",
                     "legendFormat": "limit",
                     "legendLink": null
                  }
               ],
               "title": "CPU",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "bytes"
                  },
                  "overrides": [
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "request"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#FFC000",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     },
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "limit"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#E02F44",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     }
                  ]
               },
               "id": 14,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  },
                  {
                     "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\", resource=\"memory\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "request",
                     "legendLink": null
                  },
                  {
                     "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "limit",
                     "legendLink": null
                  }
               ],
               "title": "Memory (workingset)",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "bytes"
                  },
                  "overrides": [ ]
               },
               "id": 15,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/index-gateway\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  }
               ],
               "title": "Memory (go heap inuse)",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 100,
                        "lineWidth": 0,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "normal"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "Bps"
                  },
                  "overrides": [ ]
               },
               "id": 16,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "sum by(instance, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n",
                     "format": "time_series",
                     "legendFormat": "{{pod}} - {{device}}",
                     "legendLink": null
                  }
               ],
               "title": "Disk Writes",
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 100,
                        "lineWidth": 0,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "normal"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "Bps"
                  },
                  "overrides": [ ]
               },
               "id": 17,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "sum by(instance, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n",
                     "format": "time_series",
                     "legendFormat": "{{pod}} - {{device}}",
                     "legendLink": null
                  }
               ],
               "title": "Disk Reads",
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "percentunit"
                  },
                  "overrides": [ ]
               },
               "id": 18,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*index-gateway.*\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*index-gateway.*\"})",
                     "format": "time_series",
                     "legendFormat": "{{persistentvolumeclaim}}",
                     "legendLink": null
                  }
               ],
               "title": "Disk Space Utilization",
               "type": "timeseries"
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "Index Gateway",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "request"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#FFC000",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     },
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "limit"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#E02F44",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     }
                  ]
               },
               "id": 19,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"}[$__rate_interval]))",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  },
                  {
                     "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"cpu\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "request",
                     "legendLink": null
                  },
                  {
                     "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})",
                     "format": "time_series",
                     "legendFormat": "limit",
                     "legendLink": null
                  }
               ],
               "title": "CPU",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "bytes"
                  },
                  "overrides": [
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "request"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#FFC000",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     },
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "limit"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#E02F44",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     }
                  ]
               },
               "id": 20,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  },
                  {
                     "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"memory\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "request",
                     "legendLink": null
                  },
                  {
                     "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "limit",
                     "legendLink": null
                  }
               ],
               "title": "Memory (workingset)",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "bytes"
                  },
                  "overrides": [ ]
               },
               "id": 21,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  }
               ],
               "title": "Memory (go heap inuse)",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 100,
                        "lineWidth": 0,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "normal"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "Bps"
                  },
                  "overrides": [ ]
               },
               "id": 22,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "sum by(instance, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n",
                     "format": "time_series",
                     "legendFormat": "{{pod}} - {{device}}",
                     "legendLink": null
                  }
               ],
               "title": "Disk Writes",
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 100,
                        "lineWidth": 0,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "normal"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "Bps"
                  },
                  "overrides": [ ]
               },
               "id": 23,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "sum by(instance, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n",
                     "format": "time_series",
                     "legendFormat": "{{pod}} - {{device}}",
                     "legendLink": null
                  }
               ],
               "title": "Disk Reads",
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "percentunit"
                  },
                  "overrides": [ ]
               },
               "id": 24,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 2,
               "targets": [
                  {
                     "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*bloom-gateway.*\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*bloom-gateway.*\"})",
                     "format": "time_series",
                     "legendFormat": "{{persistentvolumeclaim}}",
                     "legendLink": null
                  }
               ],
               "title": "Disk Space Utilization",
               "type": "timeseries"
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "Bloom Gateway",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "request"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#FFC000",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     },
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "limit"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#E02F44",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     }
                  ]
               },
               "id": 25,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ingester|partition-ingester\"}[$__rate_interval]))",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  },
                  {
                     "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ingester|partition-ingester\", resource=\"cpu\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "request",
                     "legendLink": null
                  },
                  {
                     "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ingester|partition-ingester\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ingester|partition-ingester\"})",
                     "format": "time_series",
                     "legendFormat": "limit",
                     "legendLink": null
                  }
               ],
               "title": "CPU",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "bytes"
                  },
                  "overrides": [
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "request"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#FFC000",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     },
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "limit"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#E02F44",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     }
                  ]
               },
               "id": 26,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ingester|partition-ingester\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  },
                  {
                     "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ingester|partition-ingester\", resource=\"memory\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "request",
                     "legendLink": null
                  },
                  {
                     "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ingester|partition-ingester\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "limit",
                     "legendLink": null
                  }
               ],
               "title": "Memory (workingset)",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "bytes"
                  },
                  "overrides": [ ]
               },
               "id": 27,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(ingester|partition-ingester).*\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  }
               ],
               "title": "Memory (go heap inuse)",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "Ingester",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [ ]
               },
               "id": 28,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 3,
               "targets": [
                  {
                     "expr": "sum by(pod) (loki_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}) or sum by(pod) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  }
               ],
               "title": "Rules",
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "request"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#FFC000",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     },
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "limit"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#E02F44",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     }
                  ]
               },
               "id": 29,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 3,
               "targets": [
                  {
                     "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"}[$__rate_interval]))",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  },
                  {
                     "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\", resource=\"cpu\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "request",
                     "legendLink": null
                  },
                  {
                     "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"})",
                     "format": "time_series",
                     "legendFormat": "limit",
                     "legendLink": null
                  }
               ],
               "title": "CPU",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "bytes"
                  },
                  "overrides": [
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "request"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#FFC000",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     },
                     {
                        "matcher": {
                           "id": "byName",
                           "options": "limit"
                        },
                        "properties": [
                           {
                              "id": "color",
                              "value": {
                                 "fixedColor": "#E02F44",
                                 "mode": "fixed"
                              }
                           },
                           {
                              "id": "custom.fillOpacity",
                              "value": 0
                           }
                        ]
                     }
                  ]
               },
               "id": 30,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 3,
               "targets": [
                  {
                     "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  },
                  {
                     "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\", resource=\"memory\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "request",
                     "legendLink": null
                  },
                  {
                     "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"} > 0)",
                     "format": "time_series",
                     "legendFormat": "limit",
                     "legendLink": null
                  }
               ],
               "title": "Memory (workingset)",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "bytes"
                  },
                  "overrides": [ ]
               },
               "id": 31,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 3,
               "targets": [
                  {
                     "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"})",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  }
               ],
               "title": "Memory (go heap inuse)",
               "tooltip": {
                  "sort": 2
               },
               "type": "timeseries"
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "Ruler",
         "titleSize": "h6"
      }
   ],
   "schemaVersion": 14,
   "style": "dark",
   "tags": [
      "loki"
   ],
   "templating": {
      "list": [
         {
            "current": {
               "text": "default",
               "value": "default"
            },
            "hide": 0,
            "label": "Data source",
            "name": "datasource",
            "options": [ ],
            "query": "prometheus",
            "refresh": 1,
            "regex": "",
            "type": "datasource"
         },
         {
            "allValue": null,
            "current": {
               "text": "prod",
               "value": "prod"
            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": false,
            "label": "cluster",
            "multi": false,
            "name": "cluster",
            "options": [ ],
            "query": "label_values(loki_build_info, cluster)",
            "refresh": 1,
            "regex": "",
            "sort": 2,
            "tagValuesQuery": "",
            "tags": [ ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
         },
         {
            "allValue": null,
            "current": {
               "text": "prod",
               "value": "prod"
            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": false,
            "label": "namespace",
            "multi": false,
            "name": "namespace",
            "options": [ ],
            "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
            "refresh": 1,
            "regex": "",
            "sort": 2,
            "tagValuesQuery": "",
            "tags": [ ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
         }
      ]
   },
   "time": {
      "from": "now-1h",
      "to": "now"
   },
   "timepicker": {
      "refresh_intervals": [
         "5s",
         "10s",
         "30s",
         "1m",
         "5m",
         "15m",
         "30m",
         "1h",
         "2h",
         "1d"
      ],
      "time_options": [
         "5m",
         "15m",
         "1h",
         "6h",
         "12h",
         "24h",
         "2d",
         "7d",
         "30d"
      ]
   },
   "timezone": "utc",
   "title": "Loki / Reads Resources",
   "uid": "reads-resources",
   "version": 0
}