1730 lines
No EOL
41 KiB
JSON
1730 lines
No EOL
41 KiB
JSON
{
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"builtIn": 1,
|
|
"datasource": {
|
|
"type": "datasource",
|
|
"uid": "grafana"
|
|
},
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"name": "Annotations & Alerts",
|
|
"target": {
|
|
"limit": 100,
|
|
"matchAny": false,
|
|
"tags": [],
|
|
"type": "dashboard"
|
|
},
|
|
"type": "dashboard"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "datasource",
|
|
"uid": "grafana"
|
|
},
|
|
"enable": true,
|
|
"iconColor": "red",
|
|
"name": "flux events",
|
|
"target": {
|
|
"limit": 100,
|
|
"matchAny": false,
|
|
"tags": [
|
|
"flux"
|
|
],
|
|
"type": "tags"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 0,
|
|
"id": 1,
|
|
"links": [],
|
|
"liveNow": false,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 100
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"id": 24,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"justifyMode": "auto",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"text": {},
|
|
"textMode": "value"
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(go_info{namespace=\"$namespace\",pod=~\".*-controller-.*\"})",
|
|
"interval": "",
|
|
"legendFormat": "pods",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Controllers",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "#EAB839",
|
|
"value": 50
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 100
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 0
|
|
},
|
|
"id": 23,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "auto",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"text": {},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "max(workqueue_longest_running_processor_seconds{namespace=\"$namespace\",pod=~\".*-controller-.*\"})",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "seconds",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Max Work Queue",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "#EAB839",
|
|
"value": 500000000
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 900000000
|
|
}
|
|
]
|
|
},
|
|
"unit": "decbits"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 0
|
|
},
|
|
"id": 25,
|
|
"options": {
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"showThresholdLabels": false,
|
|
"showThresholdMarkers": true,
|
|
"text": {}
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(go_memstats_alloc_bytes{namespace=\"$namespace\",pod=~\".*-controller-.*\"})",
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Memory",
|
|
"type": "gauge"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "#EAB839",
|
|
"value": 50
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 100
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 0
|
|
},
|
|
"id": 26,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "auto",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"mean"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"text": {},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\",pod=~\".*-controller-.*\"}[1m]))",
|
|
"interval": "",
|
|
"legendFormat": "requests",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "API Requests",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 5
|
|
},
|
|
"id": 21,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\"}[1m]))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "total",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(rate(rest_client_requests_total{namespace=\"$namespace\",code!~\"2..\"}[1m]))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "errors",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Kubernetes API Requests",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 13
|
|
},
|
|
"id": 15,
|
|
"panels": [],
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Resource Usage",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "normal"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 14
|
|
},
|
|
"id": 11,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "rate(process_cpu_seconds_total{namespace=\"$namespace\",pod=~\".*-controller-.*\"}[1m])",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "CPU Usage",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "normal"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 14
|
|
},
|
|
"id": 13,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\",container!=\"POD\",container!=\"\",pod=~\".*-controller-.*\"}) by (pod)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Memory Usage",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 25
|
|
},
|
|
"id": 17,
|
|
"panels": [],
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Reconciliation Stats",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 26
|
|
},
|
|
"id": 27,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "workqueue_longest_running_processor_seconds{name=\"kustomization\"}",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "kustomizations",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Cluster Reconciliation Duration",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "bars",
|
|
"fillOpacity": 100,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "stepAfter",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "opm"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 34
|
|
},
|
|
"id": 2,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(increase(controller_runtime_reconcile_total{controller=\"kustomization\",result!=\"error\"}[1m])) by (controller)",
|
|
"format": "time_series",
|
|
"interval": "",
|
|
"legendFormat": "successful reconciliations ",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(increase(controller_runtime_reconcile_total{controller=\"kustomization\",result=\"error\"}[1m])) by (controller)",
|
|
"format": "time_series",
|
|
"interval": "",
|
|
"legendFormat": "failed reconciliations ",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Cluster Reconciliations ops/min",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 42
|
|
},
|
|
"id": 29,
|
|
"panels": [],
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Sources Stats",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "bars",
|
|
"fillOpacity": 100,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "stepAfter",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "opm"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 43
|
|
},
|
|
"id": 4,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(increase(controller_runtime_reconcile_total{controller=\"gitrepository\",result!=\"error\"}[1m]))",
|
|
"format": "time_series",
|
|
"interval": "",
|
|
"legendFormat": "successful git pulls",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(increase(controller_runtime_reconcile_total{controller=\"gitrepository\",result=\"error\"}[1m]))",
|
|
"format": "time_series",
|
|
"interval": "",
|
|
"legendFormat": "failed git pulls",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Git Repos ops/min",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "bars",
|
|
"fillOpacity": 100,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "stepAfter",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "opm"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 43
|
|
},
|
|
"id": 30,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(increase(controller_runtime_reconcile_total{controller=\"ocirepository\",result!=\"error\"}[1m]))",
|
|
"format": "time_series",
|
|
"interval": "",
|
|
"legendFormat": "successful oci pulls",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(increase(controller_runtime_reconcile_total{controller=\"ocirepository\",result=\"error\"}[1m]))",
|
|
"format": "time_series",
|
|
"interval": "",
|
|
"legendFormat": "failed oci pulls",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "OCI Repos ops/min",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "bars",
|
|
"fillOpacity": 100,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "stepAfter",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "opm"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 52
|
|
},
|
|
"id": 31,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrepository\",result!=\"error\"}[1m]))",
|
|
"format": "time_series",
|
|
"interval": "",
|
|
"legendFormat": "successful helm pulls",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrepository\",result=\"error\"}[1m]))",
|
|
"format": "time_series",
|
|
"interval": "",
|
|
"legendFormat": "failed helm pulls",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Helm Repos ops/min",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "bars",
|
|
"fillOpacity": 100,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "stepAfter",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "opm"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 52
|
|
},
|
|
"id": 32,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(increase(controller_runtime_reconcile_total{controller=\"bucket\",result!=\"error\"}[1m]))",
|
|
"format": "time_series",
|
|
"interval": "",
|
|
"legendFormat": "successful bucket pulls",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(increase(controller_runtime_reconcile_total{controller=\"bucket\",result=\"error\"}[1m]))",
|
|
"format": "time_series",
|
|
"interval": "",
|
|
"legendFormat": "failed bucket pulls",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Buckets ops/min",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 61
|
|
},
|
|
"id": 19,
|
|
"panels": [],
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Helm Stats",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 62
|
|
},
|
|
"id": 9,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "right",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.50, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
|
|
"hide": true,
|
|
"interval": "",
|
|
"legendFormat": "P50",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.90, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
|
|
"hide": true,
|
|
"interval": "",
|
|
"legendFormat": "P90",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"helmrelease\"}[5m])) by (le))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "P99",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Helm Release Duration",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "bars",
|
|
"fillOpacity": 100,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "stepAfter",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "opm"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 70
|
|
},
|
|
"id": 5,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrelease\",result!=\"error\"}[1m])) by (controller)",
|
|
"format": "time_series",
|
|
"interval": "",
|
|
"legendFormat": "successful reconciliations ",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmrelease\",result=\"error\"}[1m])) by (controller)",
|
|
"format": "time_series",
|
|
"interval": "",
|
|
"legendFormat": "failed reconciliations ",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Helm Releases ops/min",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "bars",
|
|
"fillOpacity": 100,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "stepAfter",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "opm"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 70
|
|
},
|
|
"id": 6,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "10.0.3",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmchart\",result!=\"error\"}[1m])) by (controller)",
|
|
"format": "time_series",
|
|
"interval": "",
|
|
"legendFormat": "successful chart pulls",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"expr": "sum(increase(controller_runtime_reconcile_total{controller=\"helmchart\",result=\"error\"}[1m])) by (controller)",
|
|
"format": "time_series",
|
|
"interval": "",
|
|
"legendFormat": "failed chart pulls",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Helm Charts ops/min",
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"refresh": "10s",
|
|
"schemaVersion": 38,
|
|
"style": "light",
|
|
"tags": [
|
|
"flux"
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "Prometheus",
|
|
"value": "Prometheus"
|
|
},
|
|
"hide": 2,
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"name": "DS_PROMETHEUS",
|
|
"options": [],
|
|
"query": "prometheus",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"type": "datasource"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "flux-system",
|
|
"value": "flux-system"
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"definition": "workqueue_work_duration_seconds_count",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"name": "namespace",
|
|
"options": [],
|
|
"query": {
|
|
"query": "workqueue_work_duration_seconds_count",
|
|
"refId": "Prometheus-namespace-Variable-Query"
|
|
},
|
|
"refresh": 2,
|
|
"regex": "/.*namespace=\"([^\"]*).*/",
|
|
"skipUrlSync": false,
|
|
"sort": 0,
|
|
"tagValuesQuery": "",
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "now-15m",
|
|
"to": "now"
|
|
},
|
|
"timepicker": {
|
|
"refresh_intervals": [
|
|
"10s",
|
|
"30s",
|
|
"1m",
|
|
"5m",
|
|
"15m",
|
|
"30m",
|
|
"1h",
|
|
"2h",
|
|
"1d"
|
|
]
|
|
},
|
|
"timezone": "",
|
|
"title": "Flux Control Plane",
|
|
"uid": "flux-control-plane",
|
|
"version": 2,
|
|
"weekStart": ""
|
|
} |