monitoring-mixins-website/assets/loki/dashboards/loki-deletion.json

{
   "annotations": {
      "list": [ ]
   },
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
   "hideControls": false,
   "links": [
      {
         "asDropdown": true,
         "icon": "external link",
         "includeVars": true,
         "keepTime": true,
         "tags": [
            "loki"
         ],
         "targetBlank": false,
         "title": "Loki Dashboards",
         "type": "dashboards"
      }
   ],
   "refresh": "10s",
   "rows": [
      {
         "collapse": false,
         "height": "100px",
         "panels": [
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "format": "none",
               "id": 1,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 6,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum(loki_compactor_pending_delete_requests_count{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
                     "format": "time_series",
                     "instant": true,
                     "refId": "A"
                  }
               ],
               "thresholds": "70,80",
               "timeFrom": null,
               "timeShift": null,
               "title": "Number of Pending Requests",
               "tooltip": {
                  "shared": true,
                  "sort": 2,
                  "value_type": "individual"
               },
               "type": "singlestat",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "format": "dtdurations",
               "id": 2,
               "legend": {
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "show": true,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null as zero",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 6,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "max(loki_compactor_oldest_pending_delete_request_age_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
                     "format": "time_series",
                     "instant": true,
                     "refId": "A"
                  }
               ],
               "thresholds": "70,80",
               "timeFrom": null,
               "timeShift": null,
               "title": "Oldest Pending Request Age",
               "tooltip": {
                  "shared": true,
                  "sort": 2,
                  "value_type": "individual"
               },
               "type": "singlestat",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": false
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": false,
         "title": "Headlines",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [ ]
               },
               "id": 3,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on() vector(0)) - on () (loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on () vector(0))",
                     "format": "time_series",
                     "legendFormat": "in progress",
                     "legendLink": null
                  }
               ],
               "title": "# of Delete Requests (received - processed) ",
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [ ]
               },
               "id": 4,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "sum(increase(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))",
                     "format": "time_series",
                     "legendFormat": "received",
                     "legendLink": null
                  }
               ],
               "title": "Delete Requests Received / Day",
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [ ]
               },
               "id": 5,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "sum(increase(loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))",
                     "format": "time_series",
                     "legendFormat": "processed",
                     "legendLink": null
                  }
               ],
               "title": "Delete Requests Processed / Day",
               "type": "timeseries"
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "Churn",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [ ]
               },
               "id": 6,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  }
               ],
               "title": "Compactor CPU usage",
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [ ]
               },
               "id": 7,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} / 1024 / 1024 ",
                     "format": "time_series",
                     "legendFormat": " {{pod}} ",
                     "legendLink": null
                  }
               ],
               "title": "Compactor memory usage (MiB)",
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [ ]
               },
               "id": 8,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 4,
               "targets": [
                  {
                     "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}",
                     "format": "time_series",
                     "legendFormat": "{{pod}}",
                     "legendLink": null
                  }
               ],
               "title": "Compaction run duration (seconds)",
               "type": "timeseries"
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "Compactor",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [ ]
               },
               "id": 9,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 6,
               "targets": [
                  {
                     "expr": "sum(increase(loki_compactor_load_pending_requests_attempts_total{status=\"fail\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[1h]))",
                     "format": "time_series",
                     "legendFormat": "failures",
                     "legendLink": null
                  }
               ],
               "title": "Failures in Loading Delete Requests / Hour",
               "type": "timeseries"
            },
            {
               "datasource": "$datasource",
               "fieldConfig": {
                  "defaults": {
                     "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "lineWidth": 1,
                        "pointSize": 5,
                        "showPoints": "never",
                        "spanNulls": false,
                        "stacking": {
                           "group": "A",
                           "mode": "none"
                        }
                     },
                     "thresholds": {
                        "mode": "absolute",
                        "steps": [ ]
                     },
                     "unit": "short"
                  },
                  "overrides": [ ]
               },
               "id": 10,
               "links": [ ],
               "options": {
                  "legend": {
                     "showLegend": true
                  },
                  "tooltip": {
                     "mode": "single",
                     "sort": "none"
                  }
               },
               "span": 6,
               "targets": [
                  {
                     "expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}[$__rate_interval])) by (user)",
                     "format": "time_series",
                     "legendFormat": "{{user}}",
                     "legendLink": null
                  }
               ],
               "title": "Lines Deleted / Sec",
               "type": "timeseries"
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "Deletion metrics",
         "titleSize": "h6"
      },
      {
         "collapse": false,
         "height": "250px",
         "panels": [
            {
               "datasource": "$loki_datasource",
               "id": 11,
               "span": 6,
               "targets": [
                  {
                     "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
                     "refId": "A"
                  }
               ],
               "title": "In progress/finished",
               "type": "logs"
            },
            {
               "datasource": "$loki_datasource",
               "id": 12,
               "span": 6,
               "targets": [
                  {
                     "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
                     "refId": "A"
                  }
               ],
               "title": "Requests",
               "type": "logs"
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": true,
         "title": "List of deletion requests",
         "titleSize": "h6"
      }
   ],
   "schemaVersion": 14,
   "style": "dark",
   "tags": [
      "loki"
   ],
   "templating": {
      "list": [
         {
            "current": {
               "text": "default",
               "value": "default"
            },
            "hide": 0,
            "label": "Data source",
            "name": "datasource",
            "options": [ ],
            "query": "prometheus",
            "refresh": 1,
            "regex": "",
            "type": "datasource"
         },
         {
            "allValue": null,
            "current": {
               "text": "prod",
               "value": "prod"
            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": false,
            "label": "cluster",
            "multi": false,
            "name": "cluster",
            "options": [ ],
            "query": "label_values(loki_build_info, cluster)",
            "refresh": 1,
            "regex": "",
            "sort": 2,
            "tagValuesQuery": "",
            "tags": [ ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
         },
         {
            "allValue": null,
            "current": {
               "text": "prod",
               "value": "prod"
            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": false,
            "label": "namespace",
            "multi": false,
            "name": "namespace",
            "options": [ ],
            "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
            "refresh": 1,
            "regex": "",
            "sort": 2,
            "tagValuesQuery": "",
            "tags": [ ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
         },
         {
            "hide": 0,
            "label": null,
            "name": "loki_datasource",
            "options": [ ],
            "query": "loki",
            "refresh": 1,
            "regex": "",
            "type": "datasource"
         }
      ]
   },
   "time": {
      "from": "now-1h",
      "to": "now"
   },
   "timepicker": {
      "refresh_intervals": [
         "5s",
         "10s",
         "30s",
         "1m",
         "5m",
         "15m",
         "30m",
         "1h",
         "2h",
         "1d"
      ],
      "time_options": [
         "5m",
         "15m",
         "1h",
         "6h",
         "12h",
         "24h",
         "2d",
         "7d",
         "30d"
      ]
   },
   "timezone": "utc",
   "title": "Loki / Deletion",
   "uid": "deletion",
   "version": 0
}