2021-12-28 03:25:10 +00:00
{
"annotations" : {
"list" : [ ]
} ,
"editable" : true ,
"gnetId" : null ,
"graphTooltip" : 0 ,
"hideControls" : false ,
"links" : [
{
"asDropdown" : true ,
"icon" : "external link" ,
"includeVars" : true ,
"keepTime" : true ,
"tags" : [
"loki"
] ,
"targetBlank" : false ,
"title" : "Loki Dashboards" ,
"type" : "dashboards"
}
] ,
"refresh" : "10s" ,
"rows" : [
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fill" : 1 ,
"id" : 1 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-07 04:13:56 +00:00
"expr" : "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"})" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "series" ,
"legendLink" : null ,
"step" : 10
}
] ,
"thresholds" : [ ] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Series" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fill" : 1 ,
"id" : 2 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-07 04:13:56 +00:00
"expr" : "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}) / sum(loki_ingester_memory_streams{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"})" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "chunks" ,
"legendLink" : null ,
"step" : 10
}
] ,
"thresholds" : [ ] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Chunks per series" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
"title" : "Active Series / Chunks" ,
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fill" : 1 ,
"id" : 3 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-07 04:13:56 +00:00
"expr" : "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le)) * 1" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "99th Percentile" ,
"refId" : "A" ,
"step" : 10
} ,
{
2022-10-07 04:13:56 +00:00
"expr" : "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le)) * 1" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "50th Percentile" ,
"refId" : "B" ,
"step" : 10
} ,
{
2022-10-07 04:13:56 +00:00
"expr" : "sum(rate(loki_ingester_chunk_utilization_sum{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]))" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "Average" ,
"refId" : "C" ,
"step" : 10
}
] ,
"thresholds" : [ ] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Utilization" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "percentunit" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fill" : 1 ,
"id" : 4 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-07 04:13:56 +00:00
"expr" : "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le)) * 1e3" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "99th Percentile" ,
"refId" : "A" ,
"step" : 10
} ,
{
2022-10-07 04:13:56 +00:00
"expr" : "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le)) * 1e3" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "50th Percentile" ,
"refId" : "B" ,
"step" : 10
} ,
{
2022-10-07 04:13:56 +00:00
"expr" : "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]))" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "Average" ,
"refId" : "C" ,
"step" : 10
}
] ,
"thresholds" : [ ] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Age" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "ms" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
"title" : "Flush Stats" ,
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fill" : 1 ,
"id" : 5 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-07 04:13:56 +00:00
"expr" : "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le)) * 1" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "99th Percentile" ,
"refId" : "A" ,
"step" : 10
} ,
{
2022-10-07 04:13:56 +00:00
"expr" : "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le)) * 1" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "50th Percentile" ,
"refId" : "B" ,
"step" : 10
} ,
{
2022-10-07 04:13:56 +00:00
"expr" : "sum(rate(loki_ingester_chunk_entries_sum{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]))" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "Average" ,
"refId" : "C" ,
"step" : 10
}
] ,
"thresholds" : [ ] ,
"timeFrom" : null ,
"timeShift" : null ,
2022-01-06 03:26:14 +00:00
"title" : "Log Entries Per Chunk" ,
2021-12-28 03:25:10 +00:00
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fill" : 1 ,
"id" : 6 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-07 04:13:56 +00:00
"expr" : "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[5m])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[5m]))" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
2022-01-06 03:26:14 +00:00
"legendFormat" : "Index Entries" ,
2021-12-28 03:25:10 +00:00
"legendLink" : null ,
"step" : 10
}
] ,
"thresholds" : [ ] ,
"timeFrom" : null ,
"timeShift" : null ,
2022-01-06 03:26:14 +00:00
"title" : "Index Entries Per Chunk" ,
2021-12-28 03:25:10 +00:00
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
"title" : "Flush Stats" ,
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fill" : 1 ,
"id" : 7 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-07 04:13:56 +00:00
"expr" : "cortex_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"legendLink" : null ,
"step" : 10
}
] ,
"thresholds" : [ ] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Queue Length" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"aliasColors" : {
"1xx" : "#EAB839" ,
"2xx" : "#7EB26D" ,
"3xx" : "#6ED0E0" ,
"4xx" : "#EF843C" ,
"5xx" : "#E24D42" ,
"error" : "#E24D42" ,
"success" : "#7EB26D"
} ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fill" : 10 ,
"id" : 8 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 0 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : true ,
"steppedLine" : false ,
"targets" : [
{
2022-10-07 04:13:56 +00:00
"expr" : "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{status}}" ,
"refId" : "A" ,
"step" : 10
}
] ,
"thresholds" : [ ] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Flush Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
"title" : "Flush Stats" ,
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fill" : 1 ,
"id" : 9 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-07 04:13:56 +00:00
"expr" : "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]))" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"legendLink" : null ,
"step" : 10
}
] ,
"thresholds" : [ ] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Chunks Flushed/Second" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fill" : 1 ,
"id" : 10 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"span" : 6 ,
"stack" : true ,
"steppedLine" : false ,
"targets" : [
{
2022-10-07 04:13:56 +00:00
"expr" : "sum by (reason) (rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]))" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{reason}}" ,
"legendLink" : null ,
"step" : 10
}
] ,
"thresholds" : [ ] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Chunk Flush Reason" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : 1 ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : 1 ,
"min" : null ,
"show" : false
}
]
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
"title" : "Flush Stats" ,
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"cards" : {
"cardPadding" : null ,
"cardRound" : null
} ,
"color" : {
"cardColor" : "#b4ff00" ,
"colorScale" : "sqrt" ,
"colorScheme" : "interpolateSpectral" ,
"exponent" : 0.5 ,
"mode" : "spectrum"
} ,
"dataFormat" : "tsbuckets" ,
"datasource" : "$datasource" ,
"heatmap" : { } ,
"hideZeroBuckets" : false ,
"highlightCards" : true ,
"id" : 11 ,
"legend" : {
"show" : true
} ,
"span" : 12 ,
"targets" : [
{
2022-10-07 04:13:56 +00:00
"expr" : "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]))" ,
2021-12-28 03:25:10 +00:00
"format" : "heatmap" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{le}}" ,
"refId" : "A"
}
] ,
"title" : "Chunk Utilization" ,
"tooltip" : {
"show" : true ,
"showHistogram" : true
} ,
"type" : "heatmap" ,
"xAxis" : {
"show" : true
} ,
"xBucketNumber" : null ,
"xBucketSize" : null ,
"yAxis" : {
"decimals" : 0 ,
"format" : "percentunit" ,
"show" : true ,
"splitFactor" : null
} ,
"yBucketBound" : "auto"
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
2022-01-06 03:26:14 +00:00
"title" : "Utilization" ,
2021-12-28 03:25:10 +00:00
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"cards" : {
"cardPadding" : null ,
"cardRound" : null
} ,
"color" : {
"cardColor" : "#b4ff00" ,
"colorScale" : "sqrt" ,
"colorScheme" : "interpolateSpectral" ,
"exponent" : 0.5 ,
"mode" : "spectrum"
} ,
"dataFormat" : "tsbuckets" ,
"datasource" : "$datasource" ,
"heatmap" : { } ,
"hideZeroBuckets" : false ,
"highlightCards" : true ,
"id" : 12 ,
"legend" : {
"show" : true
} ,
"span" : 12 ,
"targets" : [
{
2022-10-07 04:13:56 +00:00
"expr" : "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le)" ,
2021-12-28 03:25:10 +00:00
"format" : "heatmap" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{le}}" ,
"refId" : "A"
}
] ,
"title" : "Chunk Size Bytes" ,
"tooltip" : {
"show" : true ,
"showHistogram" : true
} ,
"type" : "heatmap" ,
"xAxis" : {
"show" : true
} ,
"xBucketNumber" : null ,
"xBucketSize" : null ,
"yAxis" : {
"decimals" : 0 ,
"format" : "bytes" ,
"show" : true ,
"splitFactor" : null
} ,
"yBucketBound" : "auto"
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
"title" : "Utilization" ,
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fill" : 1 ,
"id" : 13 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"span" : 12 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-07 04:13:56 +00:00
"expr" : "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[1m])) by (le))" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "p99" ,
"legendLink" : null ,
"step" : 10
} ,
{
2022-10-07 04:13:56 +00:00
"expr" : "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[1m])) by (le))" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "p90" ,
"legendLink" : null ,
"step" : 10
} ,
{
2022-10-07 04:13:56 +00:00
"expr" : "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[1m])) by (le))" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "p50" ,
"legendLink" : null ,
"step" : 10
}
] ,
"thresholds" : [ ] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Chunk Size Quantiles" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "bytes" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
"title" : "Utilization" ,
"titleSize" : "h6"
} ,
{
"collapse" : false ,
"height" : "250px" ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : "$datasource" ,
"fill" : 1 ,
"id" : 14 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"links" : [ ] ,
"nullPointMode" : "null as zero" ,
"percentage" : false ,
"pointradius" : 5 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"span" : 12 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-07 04:13:56 +00:00
"expr" : "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[5m])) by (le))" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "p50" ,
"legendLink" : null ,
"step" : 10
} ,
{
2022-10-07 04:13:56 +00:00
"expr" : "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[5m])) by (le))" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "p99" ,
"legendLink" : null ,
"step" : 10
} ,
{
2022-10-07 04:13:56 +00:00
"expr" : "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[5m])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[5m]))" ,
2021-12-28 03:25:10 +00:00
"format" : "time_series" ,
"intervalFactor" : 2 ,
"legendFormat" : "avg" ,
"legendLink" : null ,
"step" : 10
}
] ,
"thresholds" : [ ] ,
"timeFrom" : null ,
"timeShift" : null ,
"title" : "Chunk Duration hours (end-start)" ,
"tooltip" : {
"shared" : true ,
"sort" : 2 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"buckets" : null ,
"mode" : "time" ,
"name" : null ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : 0 ,
"show" : true
} ,
{
"format" : "short" ,
"label" : null ,
"logBase" : 1 ,
"max" : null ,
"min" : null ,
"show" : false
}
]
}
] ,
"repeat" : null ,
"repeatIteration" : null ,
"repeatRowId" : null ,
"showTitle" : true ,
"title" : "Duration" ,
"titleSize" : "h6"
}
] ,
"schemaVersion" : 14 ,
"style" : "dark" ,
"tags" : [
"loki"
] ,
"templating" : {
"list" : [
{
"current" : {
"text" : "default" ,
"value" : "default"
} ,
"hide" : 0 ,
"label" : "Data Source" ,
"name" : "datasource" ,
"options" : [ ] ,
"query" : "prometheus" ,
"refresh" : 1 ,
"regex" : "" ,
"type" : "datasource"
} ,
{
"allValue" : null ,
"current" : {
"text" : "prod" ,
"value" : "prod"
} ,
"datasource" : "$datasource" ,
"hide" : 0 ,
"includeAll" : false ,
"label" : "cluster" ,
"multi" : false ,
"name" : "cluster" ,
"options" : [ ] ,
"query" : "label_values(loki_build_info, cluster)" ,
"refresh" : 1 ,
"regex" : "" ,
"sort" : 2 ,
"tagValuesQuery" : "" ,
"tags" : [ ] ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
} ,
{
"allValue" : null ,
"current" : {
"text" : "prod" ,
"value" : "prod"
} ,
"datasource" : "$datasource" ,
"hide" : 0 ,
"includeAll" : false ,
"label" : "namespace" ,
"multi" : false ,
"name" : "namespace" ,
"options" : [ ] ,
"query" : "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)" ,
"refresh" : 1 ,
"regex" : "" ,
"sort" : 2 ,
"tagValuesQuery" : "" ,
"tags" : [ ] ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
}
]
} ,
"time" : {
"from" : "now-1h" ,
"to" : "now"
} ,
"timepicker" : {
"refresh_intervals" : [
"5s" ,
"10s" ,
"30s" ,
"1m" ,
"5m" ,
"15m" ,
"30m" ,
"1h" ,
"2h" ,
"1d"
] ,
"time_options" : [
"5m" ,
"15m" ,
"1h" ,
"6h" ,
"12h" ,
"24h" ,
"2d" ,
"7d" ,
"30d"
]
} ,
"timezone" : "utc" ,
"title" : "Loki / Chunks" ,
"uid" : "chunks" ,
"version" : 0
}