diff --git a/assets/MSSQL/dashboards/mssql-overview.json b/assets/MSSQL/dashboards/mssql-overview.json index 53a4076..dad4115 100644 --- a/assets/MSSQL/dashboards/mssql-overview.json +++ b/assets/MSSQL/dashboards/mssql-overview.json @@ -183,7 +183,6 @@ "y": 0 }, "id": 3, - "interval": "1m", "options": { "legend": { "calcs": [ ], @@ -203,6 +202,7 @@ }, "expr": "rate(mssql_batch_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", + "interval": "1m", "intervalFactor": 2, "legendFormat": "{{instance}}" } @@ -275,7 +275,6 @@ "y": 8 }, "id": 4, - "interval": "1m", "options": { "legend": { "calcs": [ ], @@ -293,8 +292,9 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "increase(mssql_kill_connection_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "increase(mssql_kill_connection_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval:])", "format": "time_series", + "interval": "1m", "intervalFactor": 2, "legendFormat": "{{instance}}" } @@ -367,7 +367,6 @@ "y": 8 }, "id": 5, - "interval": "1m", "options": { "legend": { "calcs": [ ], @@ -387,6 +386,7 @@ }, "expr": "rate(mssql_deadlocks_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", + "interval": "1m", "intervalFactor": 2, "legendFormat": "{{instance}}" } @@ -758,7 +758,6 @@ "y": 41 }, "id": 11, - "interval": "1m", "options": { "legend": { "calcs": [ ], @@ -776,8 +775,9 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "increase(mssql_io_stall_seconds_total{job=~\"$job\", instance=~\"$instance\", db=~\"$database\", operation=\"write\"}[$__rate_interval])", + "expr": "increase(mssql_io_stall_seconds_total{job=~\"$job\", instance=~\"$instance\", db=~\"$database\", operation=\"write\"}[$__rate_interval:])", "format": "time_series", + "interval": "1m", "intervalFactor": 2, "legendFormat": "{{instance}} - {{db}}" } @@ -846,7 +846,6 @@ "y": 41 }, "id": 12, - "interval": "1m", "options": { "legend": { "calcs": [ ], @@ -864,8 +863,9 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "increase(mssql_io_stall_seconds_total{job=~\"$job\", instance=~\"$instance\", db=~\"$database\", operation=\"read\"}[$__rate_interval])", + "expr": "increase(mssql_io_stall_seconds_total{job=~\"$job\", instance=~\"$instance\", db=~\"$database\", operation=\"read\"}[$__rate_interval:])", "format": "time_series", + "interval": "1m", "intervalFactor": 2, "legendFormat": "{{instance}} - {{db}}" } @@ -938,7 +938,6 @@ "y": 49 }, "id": 13, - "interval": "1m", "options": { "legend": { "calcs": [ ], @@ -956,8 +955,9 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "increase(mssql_log_growths_total{job=~\"$job\", instance=~\"$instance\", db=~\"$database\"}[$__rate_interval])", + "expr": "increase(mssql_log_growths_total{job=~\"$job\", instance=~\"$instance\", db=~\"$database\"}[$__rate_interval:])", "format": "time_series", + "interval": "1m", "intervalFactor": 2, "legendFormat": "{{instance}} - {{db}}" } diff --git a/assets/MSSQL/dashboards/mssql-pages.json b/assets/MSSQL/dashboards/mssql-pages.json index e1b6943..591f415 100644 --- a/assets/MSSQL/dashboards/mssql-pages.json +++ b/assets/MSSQL/dashboards/mssql-pages.json @@ -363,7 +363,6 @@ "y": 8 }, "id": 5, - "interval": "1m", "options": { "legend": { "calcs": [ ], @@ -381,8 +380,9 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "increase(mssql_page_fault_count_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "increase(mssql_page_fault_count_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval:])", "format": "time_series", + "interval": "1m", "intervalFactor": 2, "legendFormat": "{{instance}}" } diff --git a/assets/aerospike/dashboards/aerospike-logs.json b/assets/aerospike/dashboards/aerospike-logs.json index 9142a5d..cc02ecc 100644 --- a/assets/aerospike/dashboards/aerospike-logs.json +++ b/assets/aerospike/dashboards/aerospike-logs.json @@ -142,17 +142,19 @@ }, "gridPos": { "h": 6, - "w": 24 + "w": 24, + "x": 0, + "y": 0 }, "id": 1, - "interval": "30s", + "maxDataPoints": 100, "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -182,7 +184,9 @@ }, "gridPos": { "h": 18, - "w": 24 + "w": 24, + "x": 0, + "y": 18 }, "id": 2, "options": { @@ -192,7 +196,7 @@ "showTime": false, "wrapLogMessage": true }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -207,7 +211,7 @@ } ], "refresh": "1m", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "aerospike-mixin" ], diff --git a/assets/alertmanager/alerts.yaml b/assets/alertmanager/alerts.yaml index 04fc0d9..4941d3e 100644 --- a/assets/alertmanager/alerts.yaml +++ b/assets/alertmanager/alerts.yaml @@ -36,7 +36,7 @@ groups: ( rate(alertmanager_notifications_failed_total{job="alertmanager"}[5m]) / - rate(alertmanager_notifications_total{job="alertmanager"}[5m]) + ignoring (reason) group_left rate(alertmanager_notifications_total{job="alertmanager"}[5m]) ) > 0.01 for: 5m @@ -53,7 +53,7 @@ groups: min by (job, integration) ( rate(alertmanager_notifications_failed_total{job="alertmanager", integration=~`.*`}[5m]) / - rate(alertmanager_notifications_total{job="alertmanager", integration=~`.*`}[5m]) + ignoring (reason) group_left rate(alertmanager_notifications_total{job="alertmanager", integration=~`.*`}[5m]) ) > 0.01 for: 5m @@ -70,7 +70,7 @@ groups: min by (job, integration) ( rate(alertmanager_notifications_failed_total{job="alertmanager", integration!~`.*`}[5m]) / - rate(alertmanager_notifications_total{job="alertmanager", integration!~`.*`}[5m]) + ignoring (reason) group_left rate(alertmanager_notifications_total{job="alertmanager", integration!~`.*`}[5m]) ) > 0.01 for: 5m diff --git a/assets/apache-activemq/dashboards/apache-activemq-logs.json b/assets/apache-activemq/dashboards/apache-activemq-logs.json index cae09ff..ef1fd09 100644 --- a/assets/apache-activemq/dashboards/apache-activemq-logs.json +++ b/assets/apache-activemq/dashboards/apache-activemq-logs.json @@ -142,17 +142,19 @@ }, "gridPos": { "h": 6, - "w": 24 + "w": 24, + "x": 0, + "y": 0 }, "id": 1, - "interval": "30s", + "maxDataPoints": 100, "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -182,7 +184,9 @@ }, "gridPos": { "h": 18, - "w": 24 + "w": 24, + "x": 0, + "y": 18 }, "id": 2, "options": { @@ -192,7 +196,7 @@ "showTime": false, "wrapLogMessage": true }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -207,7 +211,7 @@ } ], "refresh": "1m", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "apache-activemq-mixin" ], diff --git a/assets/apache-airflow/dashboards/apache-airflow-overview.json b/assets/apache-airflow/dashboards/apache-airflow-overview.json index 54d58b3..ce7c1cb 100644 --- a/assets/apache-airflow/dashboards/apache-airflow-overview.json +++ b/assets/apache-airflow/dashboards/apache-airflow-overview.json @@ -1264,6 +1264,28 @@ "type": "query", "useTags": false }, + { + "allValue": ".*", + "current": { }, + "datasource": { + "uid": "${prometheus_datasource}" + }, + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [ ], + "query": "label_values(airflow_scheduler_tasks_executable{job=~\"$job\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": ".+", "current": { }, diff --git a/assets/apache-couchdb/dashboards/couchdb-nodes.json b/assets/apache-couchdb/dashboards/couchdb-nodes.json index 8985bb2..fd30eae 100644 --- a/assets/apache-couchdb/dashboards/couchdb-nodes.json +++ b/assets/apache-couchdb/dashboards/couchdb-nodes.json @@ -1475,7 +1475,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{job=~\"$job\", couchdb_cluster=~\"$couchdb_cluster\", instance=~\"$instance\", filename=\"/var/log/couchdb/couchdb.log\"} |~ \"$log_level\"", + "expr": "{job=~\"$job\", couchdb_cluster=~\"$couchdb_cluster\", instance=~\"$instance\"} |= `` | (filename=~\"/var/log/couchdb/couchdb.log\" or log_type=\"couchdb\") |~ \"$log_level\"", "queryType": "range", "refId": "A" } diff --git a/assets/apache-hadoop/dashboards/apache-hadoop-datanode-overview.json b/assets/apache-hadoop/dashboards/apache-hadoop-datanode-overview.json index 09277db..6209d79 100644 --- a/assets/apache-hadoop/dashboards/apache-hadoop-datanode-overview.json +++ b/assets/apache-hadoop/dashboards/apache-hadoop-datanode-overview.json @@ -344,7 +344,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{job=~\"$job\", hadoop_cluster=~\"$hadoop_cluster\", instance=~\"$instance\", filename=~\".*/hadoop/logs/.*-datanode.*.log\"} |= ``", + "expr": "{job=~\"$job\", instance=~\"$instance\", hadoop_cluster=~\"$hadoop_cluster\"} |= `` | (filename=~\".*/hadoop/logs/.*-datanode.*.log\" or log_type=\"datanode\")", "queryType": "range", "refId": "A" } @@ -406,6 +406,28 @@ "type": "query", "useTags": false }, + { + "allValue": ".*", + "current": { }, + "datasource": { + "uid": "${prometheus_datasource}" + }, + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [ ], + "query": "label_values(hadoop_datanode_ramdiskblocksevictedwithoutread{job=~\"$job\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": ".+", "current": { }, diff --git a/assets/apache-hadoop/dashboards/apache-hadoop-namenode-overview.json b/assets/apache-hadoop/dashboards/apache-hadoop-namenode-overview.json index c3fdf41..ce12c2f 100644 --- a/assets/apache-hadoop/dashboards/apache-hadoop-namenode-overview.json +++ b/assets/apache-hadoop/dashboards/apache-hadoop-namenode-overview.json @@ -839,7 +839,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{job=~\"$job\", hadoop_cluster=~\"$hadoop_cluster\", instance=~\"$instance\", filename=~\".*/hadoop/logs/.*-namenode.*.log\"} |= ``", + "expr": "{job=~\"$job\", instance=~\"$instance\", hadoop_cluster=~\"$hadoop_cluster\"} |= `` | (filename=~\".*/hadoop/logs/.*-namenode.*.log\" or log_type=\"namenode\")", "queryType": "range", "refId": "A" } @@ -901,6 +901,28 @@ "type": "query", "useTags": false }, + { + "allValue": ".*", + "current": { }, + "datasource": { + "uid": "${prometheus_datasource}" + }, + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [ ], + "query": "label_values(hadoop_namenode_blockstotal{job=~\"$job\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": ".+", "current": { }, diff --git a/assets/apache-hadoop/dashboards/apache-hadoop-nodemanager-overview.json b/assets/apache-hadoop/dashboards/apache-hadoop-nodemanager-overview.json index 133f0fa..4d31517 100644 --- a/assets/apache-hadoop/dashboards/apache-hadoop-nodemanager-overview.json +++ b/assets/apache-hadoop/dashboards/apache-hadoop-nodemanager-overview.json @@ -1596,7 +1596,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{job=~\"$job\", hadoop_cluster=~\"$hadoop_cluster\", instance=~\"$instance\", filename=~\".*/hadoop/logs/.*-nodemanager.*.log\"} |= ``", + "expr": "{job=~\"$job\", instance=~\"$instance\", hadoop_cluster=~\"$hadoop_cluster\"} |= `` | (filename=~\".*/hadoop/logs/.*-nodemanager.*.log\" or log_type=\"nodemanager\")", "queryType": "range", "refId": "A" } @@ -1658,6 +1658,28 @@ "type": "query", "useTags": false }, + { + "allValue": ".*", + "current": { }, + "datasource": { + "uid": "${prometheus_datasource}" + }, + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [ ], + "query": "label_values(hadoop_nodemanager_availablegb{job=~\"$job\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": ".+", "current": { }, diff --git a/assets/apache-hadoop/dashboards/apache-hadoop-resourcemanager-overview.json b/assets/apache-hadoop/dashboards/apache-hadoop-resourcemanager-overview.json index bea401c..ef86f6e 100644 --- a/assets/apache-hadoop/dashboards/apache-hadoop-resourcemanager-overview.json +++ b/assets/apache-hadoop/dashboards/apache-hadoop-resourcemanager-overview.json @@ -907,7 +907,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{job=~\"$job\", hadoop_cluster=~\"$hadoop_cluster\", instance=~\"$instance\", filename=~\".*/hadoop/logs/.*-resourcemanager.*.log\"} |= ``", + "expr": "{job=~\"$job\", instance=~\"$instance\", hadoop_cluster=~\"$hadoop_cluster\"} |= `` | (filename=~\".*/hadoop/logs/.*-resourcemanager.*.log\" or log_type=\"resourcemanager\")", "queryType": "range", "refId": "A" } @@ -969,6 +969,28 @@ "type": "query", "useTags": false }, + { + "allValue": ".*", + "current": { }, + "datasource": { + "uid": "${prometheus_datasource}" + }, + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [ ], + "query": "label_values(hadoop_resourcemanager_activeapplications{job=~\"$job\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": ".+", "current": { }, diff --git a/assets/apache-hbase/dashboards/apache-hbase-logs.json b/assets/apache-hbase/dashboards/apache-hbase-logs.json index f3e4759..9b9593e 100644 --- a/assets/apache-hbase/dashboards/apache-hbase-logs.json +++ b/assets/apache-hbase/dashboards/apache-hbase-logs.json @@ -142,17 +142,19 @@ }, "gridPos": { "h": 6, - "w": 24 + "w": 24, + "x": 0, + "y": 0 }, "id": 1, - "interval": "30s", + "maxDataPoints": 100, "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -182,7 +184,9 @@ }, "gridPos": { "h": 18, - "w": 24 + "w": 24, + "x": 0, + "y": 18 }, "id": 2, "options": { @@ -192,7 +196,7 @@ "showTime": false, "wrapLogMessage": true }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -207,7 +211,7 @@ } ], "refresh": "1m", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "apache-hbase-mixin" ], diff --git a/assets/apache-mesos/dashboards/apache-mesos-overview.json b/assets/apache-mesos/dashboards/apache-mesos-overview.json index 655af54..3f22f17 100644 --- a/assets/apache-mesos/dashboards/apache-mesos-overview.json +++ b/assets/apache-mesos/dashboards/apache-mesos-overview.json @@ -1473,7 +1473,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{job=~\"$job\", filename=~\"/var/log/mesos/master/.*\", mesos_cluster=~\"$mesos_cluster\"} |= ``", + "expr": "{job=~\"$job\", instance=~\"$instance\", mesos_cluster=~\"$mesos_cluster\"} |= `` | (filename=~\"/var/log/mesos/master/.*\" or log_type=\"master\")", "queryType": "range", "refId": "A" } @@ -1509,7 +1509,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{job=~\"$job\", filename=~\"/var/log/mesos/agent/.*\", mesos_cluster=~\"$mesos_cluster\"} |= ``", + "expr": "{job=~\"$job\", instance=~\"$instance\", mesos_cluster=~\"$mesos_cluster\"} |= `` | (filename=~\"/var/log/mesos/agent/.*\" or log_type=\"agent\")", "queryType": "range", "refId": "A" } @@ -1571,6 +1571,28 @@ "type": "query", "useTags": false }, + { + "allValue": ".*", + "current": { }, + "datasource": { + "uid": "${prometheus_datasource}" + }, + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [ ], + "query": "label_values(mesos_exporter_build_info{job=~\"$job\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": ".+", "current": { }, @@ -1583,7 +1605,7 @@ "multi": true, "name": "instance", "options": [ ], - "query": "label_values(mesos_exporter_build_info{job=~\"$job\", instance=~\"$instance\"}, instance)", + "query": "label_values(mesos_exporter_build_info{job=~\"$job\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/assets/apache-solr/dashboards/apache-solr-logs-overview.json b/assets/apache-solr/dashboards/apache-solr-logs-overview.json index 3c20fa4..a6ab1bf 100644 --- a/assets/apache-solr/dashboards/apache-solr-logs-overview.json +++ b/assets/apache-solr/dashboards/apache-solr-logs-overview.json @@ -142,17 +142,19 @@ }, "gridPos": { "h": 6, - "w": 24 + "w": 24, + "x": 0, + "y": 0 }, "id": 1, - "interval": "30s", + "maxDataPoints": 100, "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -182,7 +184,9 @@ }, "gridPos": { "h": 18, - "w": 24 + "w": 24, + "x": 0, + "y": 18 }, "id": 2, "options": { @@ -192,7 +196,7 @@ "showTime": false, "wrapLogMessage": true }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -207,7 +211,7 @@ } ], "refresh": "1m", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "apache-solr-mixin" ], diff --git a/assets/apache-solr/dashboards/apache-solr-query-performance.json b/assets/apache-solr/dashboards/apache-solr-query-performance.json index 075d1a0..f87a575 100644 --- a/assets/apache-solr/dashboards/apache-solr-query-performance.json +++ b/assets/apache-solr/dashboards/apache-solr-query-performance.json @@ -1807,7 +1807,7 @@ "multi": true, "name": "cluster", "options": [ ], - "query": "label_values(cassandra_cache_size{job=~\"$job\"}, cluster)", + "query": "label_values(solr_metrics_core_errors_total{job=~\"$job\"}, cluster)", "refresh": 2, "regex": "", "sort": 0, diff --git a/assets/apache-solr/dashboards/apache-solr-resource-monitoring.json b/assets/apache-solr/dashboards/apache-solr-resource-monitoring.json index 38cc5aa..b3e4fb8 100644 --- a/assets/apache-solr/dashboards/apache-solr-resource-monitoring.json +++ b/assets/apache-solr/dashboards/apache-solr-resource-monitoring.json @@ -1303,7 +1303,7 @@ "multi": true, "name": "cluster", "options": [ ], - "query": "label_values(cassandra_cache_size{job=~\"$job\"}, cluster)", + "query": "label_values(solr_metrics_core_errors_total{job=~\"$job\"}, cluster)", "refresh": 2, "regex": "", "sort": 0, diff --git a/assets/apache-tomcat/alerts.yaml b/assets/apache-tomcat/alerts.yaml index 5d7464e..965b90b 100644 --- a/assets/apache-tomcat/alerts.yaml +++ b/assets/apache-tomcat/alerts.yaml @@ -8,7 +8,7 @@ groups: 80 percent. summary: The instance has a CPU usage higher than the configured threshold. expr: | - sum by (job, instance) (jvm_process_cpu_load{job=~"integrations/tomcat"}) > 80 + sum by (job,instance) (jvm_process_cpu_load{job=~"integrations/tomcat"}) > 80 for: 5m labels: severity: critical @@ -19,7 +19,7 @@ groups: of 80 percent. summary: The instance has a higher memory usage than the configured threshold. expr: | - sum(jvm_memory_usage_used_bytes{job=~"integrations/tomcat"}) by (job, instance) / sum(jvm_physical_memory_bytes{job=~"integrations/tomcat"}) by (job, instance) * 100 > 80 + sum(jvm_memory_usage_used_bytes{job=~"integrations/tomcat"}) by (job,instance) / sum(jvm_physical_memory_bytes{job=~"integrations/tomcat"}) by (job,instance) * 100 > 80 for: 5m labels: severity: critical @@ -30,7 +30,7 @@ groups: the threshold of 5 percent. summary: There are a high number of request errors. expr: | - sum by (job, instance) (increase(tomcat_errorcount_total[5m]) / increase(tomcat_requestcount_total[5m]) * 100) > 5 + sum by (job,instance) (increase(tomcat_errorcount_total{job=~"integrations/tomcat"}[5m]) / increase(tomcat_requestcount_total{job=~"integrations/tomcat"}[5m]) * 100) > 5 for: 5m labels: severity: critical @@ -41,7 +41,7 @@ groups: 300ms. summary: The processing time has been moderately high. expr: | - sum by (job, instance) (increase(tomcat_processingtime_total[5m]) / increase(tomcat_requestcount_total[5m])) > 300 + sum by (job,instance) (increase(tomcat_processingtime_total{job=~"integrations/tomcat"}[5m]) / increase(tomcat_requestcount_total{job=~"integrations/tomcat"}[5m])) > 300 for: 5m labels: severity: warning diff --git a/assets/apache-tomcat/dashboards/apache-tomcat-hosts.json b/assets/apache-tomcat/dashboards/apache-tomcat-hosts.json index 1c215ef..3cda935 100644 --- a/assets/apache-tomcat/dashboards/apache-tomcat-hosts.json +++ b/assets/apache-tomcat/dashboards/apache-tomcat-hosts.json @@ -370,7 +370,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "sum(rate(tomcat_servlet_requestcount_total{instance=~\"$instance\", job=~\"$job\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__rate_interval])) by (job, instance)", + "expr": "sum(rate(tomcat_servlet_requestcount_total{job=~\"$job\", instance=~\"$instance\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__rate_interval])) by (job, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - total requests" @@ -379,7 +379,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "sum(rate(tomcat_servlet_errorcount_total{instance=~\"$instance\", job=~\"$job\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__rate_interval])) by (job, instance)", + "expr": "sum(rate(tomcat_servlet_errorcount_total{job=~\"$job\", instance=~\"$instance\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__rate_interval])) by (job, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - total errors" @@ -388,7 +388,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "rate(tomcat_servlet_requestcount_total{instance=~\"$instance\", job=~\"$job\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__rate_interval])", + "expr": "rate(tomcat_servlet_requestcount_total{job=~\"$job\", instance=~\"$instance\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{module}}{{servlet}} - requests" @@ -397,7 +397,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "rate(tomcat_servlet_errorcount_total{instance=~\"$instance\", job=~\"$job\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__rate_interval])", + "expr": "rate(tomcat_servlet_errorcount_total{job=~\"$job\", instance=~\"$instance\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{module}}{{servlet}} - errors" @@ -488,7 +488,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "sum(increase(tomcat_servlet_processingtime_total{instance=~\"$instance\", job=~\"$job\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__interval:] offset -$__interval) / clamp_min(increase(tomcat_servlet_requestcount_total{instance=~\"$instance\", job=~\"$job\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__interval:] offset -$__interval), 1)) by (job, instance)", + "expr": "sum(increase(tomcat_servlet_processingtime_total{job=~\"$job\", instance=~\"$instance\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__interval:] offset -$__interval) / clamp_min(increase(tomcat_servlet_requestcount_total{job=~\"$job\", instance=~\"$instance\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__interval:] offset -$__interval), 1)) by (job, instance)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -498,7 +498,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "increase(tomcat_servlet_processingtime_total{instance=~\"$instance\", job=~\"$job\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__interval:] offset -$__interval) / clamp_min(increase(tomcat_servlet_requestcount_total{instance=~\"$instance\", job=~\"$job\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__interval:] offset -$__interval), 1)", + "expr": "increase(tomcat_servlet_processingtime_total{job=~\"$job\", instance=~\"$instance\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__interval:] offset -$__interval) / clamp_min(increase(tomcat_servlet_requestcount_total{job=~\"$job\", instance=~\"$instance\", module=~\"$host$context\", servlet=~\"$servlet\"}[$__interval:] offset -$__interval), 1)", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -551,6 +551,28 @@ "type": "query", "useTags": false }, + { + "allValue": ".*", + "current": { }, + "datasource": { + "uid": "${prometheus_datasource}" + }, + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [ ], + "query": "label_values(tomcat_bytesreceived_total{job=~\"$job\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": "", "current": { }, @@ -563,7 +585,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(tomcat_bytesreceived_total, instance)", + "query": "label_values(tomcat_bytesreceived_total{job=~\"$job\"}, instance)", "refresh": 1, "regex": "", "sort": 0, @@ -585,7 +607,7 @@ "multi": true, "name": "host", "options": [ ], - "query": "label_values(tomcat_session_sessioncounter_total{instance=~\"$instance\"}, host)", + "query": "label_values(tomcat_session_sessioncounter_total{job=~\"$job\"}, host)", "refresh": 1, "regex": "", "sort": 0, @@ -607,7 +629,7 @@ "multi": true, "name": "context", "options": [ ], - "query": "label_values(tomcat_session_sessioncounter_total{host=~\"$host\"}, context)", + "query": "label_values(tomcat_session_sessioncounter_total{job=~\"$job\", host=~\"$host\"}, context)", "refresh": 1, "regex": "", "sort": 0, @@ -629,7 +651,7 @@ "multi": true, "name": "servlet", "options": [ ], - "query": "label_values(tomcat_servlet_requestcount_total{module=~\"$host$context\"}, servlet)", + "query": "label_values(tomcat_servlet_requestcount_total{job=~\"$job\", module=~\"$host$context\"}, servlet)", "refresh": 1, "regex": "", "sort": 0, diff --git a/assets/apache-tomcat/dashboards/apache-tomcat-overview.json b/assets/apache-tomcat/dashboards/apache-tomcat-overview.json index a5d0f41..dd5004d 100644 --- a/assets/apache-tomcat/dashboards/apache-tomcat-overview.json +++ b/assets/apache-tomcat/dashboards/apache-tomcat-overview.json @@ -813,7 +813,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{filename=~\"/var/log/tomcat.*/catalina.out|/opt/tomcat/logs/catalina.out|/Program Files/Apache Software Foundation/Tomcat .*..*/logs/catalina.out\",job=~\"$job\", instance=~\"$instance\"} |= ``", + "expr": "{job=~\"$job\", instance=~\"$instance\"} |= `` | (filename=~\"/var/log/tomcat.*/catalina.out|/opt/tomcat/logs/catalina.out|/Program Files/Apache Software Foundation/Tomcat .*..*/logs/catalina.out\" or log_type=\"catalina.out\")", "queryType": "range", "refId": "A" } @@ -876,19 +876,19 @@ "useTags": false }, { - "allValue": "", + "allValue": ".*", "current": { }, "datasource": { "uid": "${prometheus_datasource}" }, - "hide": 0, - "includeAll": false, - "label": "Instance", - "multi": false, - "name": "instance", + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", "options": [ ], - "query": "label_values(tomcat_bytesreceived_total, instance)", - "refresh": 1, + "query": "label_values(tomcat_bytesreceived_total{job=~\"$job\"}, cluster)", + "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", @@ -898,7 +898,29 @@ "useTags": false }, { - "allValue": "", + "allValue": ".+", + "current": { }, + "datasource": { + "uid": "${prometheus_datasource}" + }, + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [ ], + "query": "label_values(tomcat_bytesreceived_total{job=~\"$job\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", "current": { }, "datasource": { "uid": "${prometheus_datasource}" @@ -909,7 +931,7 @@ "multi": true, "name": "protocol", "options": [ ], - "query": "label_values(tomcat_bytesreceived_total, protocol)", + "query": "label_values(tomcat_bytesreceived_total{job=~\"$job\"}, protocol)", "refresh": 1, "regex": "", "sort": 0, @@ -920,7 +942,7 @@ "useTags": false }, { - "allValue": "", + "allValue": ".+", "current": { }, "datasource": { "uid": "${prometheus_datasource}" @@ -931,7 +953,7 @@ "multi": true, "name": "port", "options": [ ], - "query": "label_values(tomcat_bytesreceived_total, port)", + "query": "label_values(tomcat_bytesreceived_total{job=~\"$job\"}, port)", "refresh": 1, "regex": "", "sort": 0, diff --git a/assets/blackbox_exporter/dashboards/blackbox-exporter.json b/assets/blackbox_exporter/dashboards/blackbox-exporter.json index 5335e5c..2f9dc58 100644 --- a/assets/blackbox_exporter/dashboards/blackbox-exporter.json +++ b/assets/blackbox_exporter/dashboards/blackbox-exporter.json @@ -1,7 +1,7 @@ { "__inputs": [ ], "__requires": [ ], - "description": "A dashboard that monitors the Blackbox-exporter. It is created using the blackbox-exporter-mixin for the the (blackbox-exporter)[https://github.com/prometheus/blackbox-exporter].", + "description": "A dashboard that monitors the Blackbox-exporter. It is created using the [blackbox-exporter-mixin](https://github.com/adinhodovic/blackbox-exporter-mixin) for the the (blackbox-exporter)[https://github.com/prometheus/blackbox_exporter].", "editable": true, "panels": [ { @@ -70,7 +70,7 @@ }, "textMode": "value_and_name" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -120,7 +120,7 @@ ] } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -173,7 +173,7 @@ ] } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -222,7 +222,7 @@ ] } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -259,7 +259,7 @@ ] } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -326,7 +326,7 @@ ] } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -379,7 +379,7 @@ ] } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -437,7 +437,7 @@ ] } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -491,7 +491,7 @@ ] } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -542,7 +542,7 @@ }, "textMode": "name" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -590,7 +590,7 @@ "colorMode": "background", "graphMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -643,7 +643,7 @@ ] } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -681,7 +681,7 @@ ] } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -720,7 +720,7 @@ ] } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -757,7 +757,7 @@ ] } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -808,7 +808,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -871,7 +871,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -894,7 +894,7 @@ "type": "timeseries" } ], - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "blackbox-exporter", "blackbox-exporter-mixin" diff --git a/assets/cert-manager/dashboards/cert-manager.json b/assets/cert-manager/dashboards/cert-manager.json deleted file mode 100644 index c109e39..0000000 --- a/assets/cert-manager/dashboards/cert-manager.json +++ /dev/null @@ -1,1261 +0,0 @@ -{ - "_config": { - "certManagerCertExpiryDays": "21", - "certManagerJobLabel": "cert-manager", - "certManagerRunbookURLPattern": "https://github.com/imusmanmalik/cert-manager-mixin/blob/main/RUNBOOK.md#%s", - "dashboards": { - "certmanagerCertificateExpirationTimestampSecondsSelector": "", - "certmanagerCertificateReadyStatusSelector": "", - "certmanagerControllerSyncCallCountSelector": "", - "certmanagerHttpAcmeClientRequestCountSelector": "", - "certmanagerHttpAcmeClientRequestDurationSecondsCountSelector": "", - "certmanagerHttpAcmeClientRequestDurationSecondsSumSelector": "", - "clusterVariableSelector": "", - "containerCPUUsageSecondsTotalSelector": "container=\"cert-manager\"", - "containerCpuCfsPeriodsTotalSelector": "container=\"cert-manager\"", - "containerCpuCfsThrottledPeriodsTotalSelector": "container=\"cert-manager\"", - "containerMemoryUsageBytesSelector": "container=\"cert-manager\"", - "containerNetworkReceiveBytesTotalSelector": "namespace=\"cert-manager\"", - "containerNetworkTransmitBytesTotalSelector": "namespace=\"cert-manager\"", - "containerSelector": "container=\"cert-manager\"", - "defaultSelector": "", - "enableMultiCluster": false, - "kubePodContainerResourceLimitsCpuCoresSelector": "container=\"cert-manager\"", - "kubePodContainerResourceLimitsMemoryBytesSelector": "container=\"cert-manager\"", - "kubePodContainerResourceRequestsCpuCoresSelector": "container=\"cert-manager\"", - "kubePodContainerResourceRequestsMemoryBytesSelector": "container=\"cert-manager\"", - "namespaceSelector": "namespace=\"cert-manager\"" - }, - "grafanaExternalUrl": "https://grafana.example.com" - }, - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "", - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "id": 59, - "iteration": 1616445892702, - "links": [ ], - "panels": [ - { - "datasource": "$datasource", - "description": "The number of certificates in the ready state.", - "fieldConfig": { - "defaults": { - "custom": { }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "True" - }, - "properties": [ - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto" - }, - "pluginVersion": "7.4.5", - "targets": [ - { - "expr": "sum by (condition) (certmanager_certificate_ready_status{ })", - "interval": "", - "legendFormat": "{{ condition }}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Certificates Ready", - "type": "stat" - }, - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { }, - "decimals": 1, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "#EAB839", - "value": 604800 - }, - { - "color": "green", - "value": 1209600 - } - ] - }, - "unit": "dtdurations" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto" - }, - "pluginVersion": "7.4.5", - "targets": [ - { - "expr": "min(certmanager_certificate_expiration_timestamp_seconds{ } > 0) - time()", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "expr": "vector(1250000)", - "hide": true, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Soonest Cert Expiry", - "type": "stat" - }, - { - "datasource": "$datasource", - "description": "Status of the certificates. Values are True, False or Unknown.", - "fieldConfig": { - "defaults": { - "custom": { - "align": null, - "filterable": false - }, - "mappings": [ - { - "from": "", - "id": 0, - "operator": "", - "text": "Yes", - "to": "", - "type": 1, - "value": "" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Ready Status" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Valid Until" - }, - "properties": [ - { - "id": "unit", - "value": "dateTimeAsIso" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Valid Until" - }, - "properties": [ - { - "id": "unit", - "value": "dateTimeAsIso" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 9, - "options": { - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "Valid Until" - } - ] - }, - "pluginVersion": "7.4.5", - "targets": [ - { - "expr": "label_join(avg by (name, namespace, condition, exported_namespace) (certmanager_certificate_ready_status{ } == 1), \"namespaced_name\", \"-\", \"namespace\", \"exported_namespace\", \"name\")", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "expr": "label_join(avg by (name, namespace, exported_namespace) (certmanager_certificate_expiration_timestamp_seconds{ }) * 1000, \"namespaced_name\", \"-\", \"namespace\", \"exported_namespace\", \"name\")", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Certificates", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "namespaced_name" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Value #A": true, - "exported_namespace": false, - "exported_namespace 1": false, - "exported_namespace 2": true, - "name 1": true, - "namespace 2": true, - "namespaced_name": true - }, - "indexByName": { - "Time 1": 8, - "Time 2": 10, - "Value #A": 6, - "Value #B": 5, - "condition": 4, - "exported_namespace 1": 1, - "exported_namespace 2": 11, - "name 1": 9, - "name 2": 3, - "namespace": 0, - "namespace 1": 2, - "namespaced_name": 7 - }, - "renameByName": { - "Time 1": "", - "Value #B": "Valid Until", - "condition": "Ready Status", - "exported_namespace": "Certificate Namespace", - "exported_namespace 1": "Certificate Namespace", - "exported_namespace 2": "", - "name": "Certificate", - "name 2": "Certificate", - "namespace": "Namespace", - "namespace 1": "Namespace", - "namespaced_name": "" - } - } - } - ], - "type": "table" - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "The rate of controller sync requests.", - "fieldConfig": { - "defaults": { - "custom": { }, - "links": [ ] - }, - "overrides": [ ] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "hiddenSeries": false, - "id": 7, - "interval": "20s", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "maxDataPoints": 250, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (controller) (\n rate(certmanager_controller_sync_call_count{ }[$__rate_interval ])\n)", - "interval": "", - "legendFormat": "{{controller }}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeRegions": [ ], - "timeShift": null, - "title": "Controller Sync Requests/sec", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "reqps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Rate of requests to ACME provider.", - "fieldConfig": { - "defaults": { - "custom": { }, - "links": [ ] - }, - "overrides": [ ] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "hiddenSeries": false, - "id": 6, - "interval": "20s", - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "maxDataPoints": 250, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (method, path, status) (\n rate(certmanager_http_acme_client_request_count{ }[$__rate_interval ])\n)", - "interval": "", - "legendFormat": "{{method }} {{path }} {{status }}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeRegions": [ ], - "timeShift": null, - "title": "ACME HTTP Requests/sec", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "reqps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Average duration of requests to ACME provider. ", - "fieldConfig": { - "defaults": { - "custom": { }, - "links": [ ] - }, - "overrides": [ ] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "hiddenSeries": false, - "id": 10, - "interval": "30s", - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "maxDataPoints": 250, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (method, path, status) (rate(certmanager_http_acme_client_request_duration_seconds_sum{ }[$__rate_interval ]))\n/\nsum by (method, path, status) (rate(certmanager_http_acme_client_request_duration_seconds_count{ }[$__rate_interval ]))", - "interval": "", - "legendFormat": "{{method }} {{path }} {{status }}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeRegions": [ ], - "timeShift": null, - "title": "ACME HTTP Request avg duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "max": "dark-yellow" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "CPU Usage and limits, as percent of a vCPU core. ", - "fieldConfig": { - "defaults": { - "custom": { }, - "links": [ ] - }, - "overrides": [ ] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 24 - }, - "hiddenSeries": false, - "id": 12, - "interval": "1m", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "maxDataPoints": 250, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "CPU", - "fill": 1, - "fillGradient": 5 - }, - { - "alias": "/Request.*/", - "color": "#FF9830", - "dashes": true - }, - { - "alias": "/Limit.*/", - "color": "#F2495C", - "dashes": true - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg by (pod) (rate(container_cpu_usage_seconds_total{container=\"cert-manager\" }[$__rate_interval ]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "CPU {{pod }}", - "refId": "A" - }, - { - "expr": "avg by (pod) (kube_pod_container_resource_limits_cpu_cores{container=\"cert-manager\" })", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Limit {{pod }}", - "refId": "B" - }, - { - "expr": "avg by (pod) (kube_pod_container_resource_requests_cpu_cores{container=\"cert-manager\" })", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Request {{pod }}", - "refId": "C" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeRegions": [ ], - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "max": "dark-yellow" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Percent of the time that the CPU is being throttled. Higher is badderer. ", - "fieldConfig": { - "defaults": { - "custom": { }, - "links": [ ] - }, - "overrides": [ ] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 24 - }, - "hiddenSeries": false, - "id": 14, - "interval": "1m", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "maxDataPoints": 250, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/external-dns.*/", - "fill": 1, - "fillGradient": 5 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg by (pod) (\n rate(container_cpu_cfs_throttled_periods_total{container=\"cert-manager\" }[$__rate_interval ])\n /\n rate(container_cpu_cfs_periods_total{container=\"cert-manager\" }[$__rate_interval ])\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod }}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeRegions": [ ], - "timeShift": null, - "title": "CPU Throttling", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "max": "dark-yellow" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Memory utilisation and limits.", - "fieldConfig": { - "defaults": { - "custom": { }, - "links": [ ] - }, - "overrides": [ ] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 24 - }, - "hiddenSeries": false, - "id": 16, - "interval": "1m", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "maxDataPoints": 250, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "Memory", - "fill": 1, - "fillGradient": 5 - }, - { - "alias": "Request", - "color": "#FF9830", - "dashes": true - }, - { - "alias": "Limit", - "color": "#F2495C", - "dashes": true - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg by (pod) (container_memory_usage_bytes{container=\"cert-manager\" })", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Memory {{pod }}", - "refId": "A" - }, - { - "expr": "avg by (pod) (kube_pod_container_resource_limits_memory_bytes{container=\"cert-manager\" })", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Limit {{pod }}", - "refId": "B" - }, - { - "expr": "avg by (pod) (kube_pod_container_resource_requests_memory_bytes{container=\"cert-manager\" })", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Request {{pod }}", - "refId": "C" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeRegions": [ ], - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "max": "dark-yellow" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Network ingress/egress.", - "fieldConfig": { - "defaults": { - "custom": { }, - "links": [ ] - }, - "overrides": [ ] - }, - "fill": 1, - "fillGradient": 5, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 24 - }, - "hiddenSeries": false, - "id": 18, - "interval": "1m", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "transmit", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(\n sum without (interface) (\n rate(container_network_receive_bytes_total{namespace=\"cert-manager\" }[$__rate_interval ])\n )\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "receive", - "refId": "A" - }, - { - "expr": "avg(\n sum without (interface) (\n rate(container_network_transmit_bytes_total{namespace=\"cert-manager\" }[$__rate_interval ])\n )\n)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "transmit", - "refId": "B" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeRegions": [ ], - "timeShift": null, - "title": "Network", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "1m", - "schemaVersion": 27, - "style": "dark", - "tags": [ - "cert-manager", - "infra" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "prometheus", - "value": "prometheus" - }, - "description": null, - "error": null, - "hide": 0, - "includeAll": false, - "label": "Data Source", - "multi": false, - "name": "datasource", - "options": [ ], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": false, - "text": "", - "value": "" - }, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "definition": "", - "hide": 2, - "includeAll": false, - "multi": false, - "name": "cluster", - "options": [ ], - "query": { - "query": "label_values(certmanager_certificate_ready_status{ }, cluster)", - "refId": "Prometheus-cluster-Variable-Query" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Cert Manager", - "uid": "TvuRo2iMk", - "version": 1 -} diff --git a/assets/cert-manager/dashboards/overview.json b/assets/cert-manager/dashboards/overview.json new file mode 100644 index 0000000..1f81e38 --- /dev/null +++ b/assets/cert-manager/dashboards/overview.json @@ -0,0 +1,1533 @@ +{ + "__elements": { }, + "__inputs": [ ], + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "10.4.0" + }, + { + "id": "prometheus", + "name": "Prometheus", + "type": "datasource", + "version": "1.0.0" + }, + { + "id": "stat", + "name": "Stat", + "type": "panel", + "version": "" + }, + { + "id": "table", + "name": "Table", + "type": "panel", + "version": "" + }, + { + "id": "timeseries", + "name": "Time series", + "type": "panel", + "version": "" + } + ], + "_config": { + "certManagerCertExpiryDays": "21", + "certManagerJobLabel": "cert-manager", + "certManagerRunbookURLPattern": "https://github.com/imusmanmalik/cert-manager-mixin/blob/main/RUNBOOK.md#%s", + "dashboards": { + "certmanagerCertificateExpirationTimestampSecondsSelector": "", + "certmanagerCertificateReadyStatusSelector": "", + "certmanagerControllerSyncCallCountSelector": "", + "certmanagerHttpAcmeClientRequestCountSelector": "", + "certmanagerHttpAcmeClientRequestDurationSecondsCountSelector": "", + "certmanagerHttpAcmeClientRequestDurationSecondsSumSelector": "", + "clusterVariableSelector": "", + "containerCPUUsageSecondsTotalSelector": "container=\"cert-manager\"", + "containerCpuCfsPeriodsTotalSelector": "container=\"cert-manager\"", + "containerCpuCfsThrottledPeriodsTotalSelector": "container=\"cert-manager\"", + "containerMemoryUsageBytesSelector": "container=\"cert-manager\"", + "containerNetworkReceiveBytesTotalSelector": "namespace=\"cert-manager\"", + "containerNetworkTransmitBytesTotalSelector": "namespace=\"cert-manager\"", + "containerSelector": "container=\"cert-manager\"", + "defaultSelector": "", + "enableMultiCluster": false, + "kubePodContainerResourceLimitsCpuCoresSelector": "container=\"cert-manager\"", + "kubePodContainerResourceLimitsMemoryBytesSelector": "container=\"cert-manager\"", + "kubePodContainerResourceRequestsCpuCoresSelector": "container=\"cert-manager\"", + "kubePodContainerResourceRequestsMemoryBytesSelector": "container=\"cert-manager\"", + "namespaceSelector": "namespace=\"cert-manager\"" + }, + "grafanaExternalUrl": "https://grafana.example.com" + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ ], + "panels": [ + { + "datasource": { + "uid": "$datasource" + }, + "description": "The number of certificates in the ready state.", + "fieldConfig": { + "defaults": { + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "True" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum by (condition) (certmanager_certificate_ready_status{})", + "interval": "", + "legendFormat": "{{ condition }}", + "refId": "A" + } + ], + "title": "Certificates Ready", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 604800 + }, + { + "color": "green", + "value": 1209600 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "min(certmanager_certificate_expiration_timestamp_seconds{} > 0) - time()", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "vector(1250000)", + "hide": true, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "Soonest Cert Expiry", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Status of the certificates. Values are True, False or Unknown.", + "fieldConfig": { + "defaults": { + "custom": { + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [ + { + "options": { + "": { + "text": "Yes" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Ready Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Valid Until" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Valid Until" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 9, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Valid Until" + } + ] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "label_join(avg by (name, namespace, condition, exported_namespace) (certmanager_certificate_ready_status{} == 1), \"namespaced_name\", \"-\", \"namespace\", \"exported_namespace\", \"name\")", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "label_join(avg by (name, namespace, exported_namespace) (certmanager_certificate_expiration_timestamp_seconds{}) * 1000, \"namespaced_name\", \"-\", \"namespace\", \"exported_namespace\", \"name\")", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "Certificates", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "namespaced_name" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Value #A": true, + "exported_namespace": false, + "exported_namespace 1": false, + "exported_namespace 2": true, + "name 1": true, + "namespace 2": true, + "namespaced_name": true + }, + "indexByName": { + "Time 1": 8, + "Time 2": 10, + "Value #A": 6, + "Value #B": 5, + "condition": 4, + "exported_namespace 1": 1, + "exported_namespace 2": 11, + "name 1": 9, + "name 2": 3, + "namespace": 0, + "namespace 1": 2, + "namespaced_name": 7 + }, + "renameByName": { + "Time 1": "", + "Value #B": "Valid Until", + "condition": "Ready Status", + "exported_namespace": "Certificate Namespace", + "exported_namespace 1": "Certificate Namespace", + "exported_namespace 2": "", + "name": "Certificate", + "name 2": "Certificate", + "namespace": "Namespace", + "namespace 1": "Namespace", + "namespaced_name": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "The rate of controller sync requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ ], + "mappings": [ ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 7, + "interval": "20s", + "maxDataPoints": 250, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum by (controller) (\n rate(certmanager_controller_sync_call_count{}[$__rate_interval ])\n)", + "interval": "", + "legendFormat": "{{ controller }}", + "refId": "A" + } + ], + "title": "Controller Sync Requests/sec", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Rate of requests to ACME provider.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ ], + "mappings": [ ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 6, + "interval": "20s", + "maxDataPoints": 250, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum by (method, path, status) (\n rate(certmanager_http_acme_client_request_count{}[$__rate_interval ])\n)", + "interval": "", + "legendFormat": "{{ method }} {{ path }} {{ status }}", + "refId": "A" + } + ], + "title": "ACME HTTP Requests/sec", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Average duration of requests to ACME provider. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ ], + "mappings": [ ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 10, + "interval": "30s", + "maxDataPoints": 250, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum by (method, path, status) (rate(certmanager_http_acme_client_request_duration_seconds_sum{}[$__rate_interval ]))\n/\nsum by (method, path, status) (rate(certmanager_http_acme_client_request_duration_seconds_count{}[$__rate_interval ]))", + "interval": "", + "legendFormat": "{{ method }} {{ path }} {{ status }}", + "refId": "A" + } + ], + "title": "ACME HTTP Request avg duration", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "CPU Usage and limits, as percent of a vCPU core. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ ], + "mappings": [ ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CPU" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 10 + }, + { + "id": "custom.fillOpacity", + "value": 50 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/Request.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/Limit.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 24 + }, + "id": 12, + "interval": "1m", + "maxDataPoints": 250, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg by (pod) (rate(container_cpu_usage_seconds_total{container=\"cert-manager\"}[$__rate_interval ]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "CPU {{ pod }}", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg by (pod) (kube_pod_container_resource_limits_cpu_cores{container=\"cert-manager\"})", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Limit {{ pod }}", + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg by (pod) (kube_pod_container_resource_requests_cpu_cores{container=\"cert-manager\"})", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Request {{ pod }}", + "refId": "C" + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Percent of the time that the CPU is being throttled. Higher is badderer. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ ], + "mappings": [ ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/external-dns.*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 10 + }, + { + "id": "custom.fillOpacity", + "value": 50 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 24 + }, + "id": 14, + "interval": "1m", + "maxDataPoints": 250, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg by (pod) (\n rate(container_cpu_cfs_throttled_periods_total{container=\"cert-manager\"}[$__rate_interval ])\n /\n rate(container_cpu_cfs_periods_total{container=\"cert-manager\"}[$__rate_interval ])\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ pod }}", + "refId": "A" + } + ], + "title": "CPU Throttling", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Memory utilisation and limits.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ ], + "mappings": [ ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 10 + }, + { + "id": "custom.fillOpacity", + "value": 50 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 24 + }, + "id": 16, + "interval": "1m", + "maxDataPoints": 250, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg by (pod) (container_memory_usage_bytes{container=\"cert-manager\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Memory {{ pod }}", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg by (pod) (kube_pod_container_resource_limits_memory_bytes{container=\"cert-manager\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Limit {{ pod }}", + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg by (pod) (kube_pod_container_resource_requests_memory_bytes{container=\"cert-manager\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Request {{ pod }}", + "refId": "C" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Network ingress/egress.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 24 + }, + "id": 18, + "interval": "1m", + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg(\n sum without (interface) (\n rate(container_network_receive_bytes_total{namespace=\"cert-manager\"}[$__rate_interval ])\n )\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "receive", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg(\n sum without (interface) (\n rate(container_network_transmit_bytes_total{namespace=\"cert-manager\"}[$__rate_interval ])\n )\n)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "transmit", + "refId": "B" + } + ], + "title": "Network", + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "cert-manager", + "infra" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [ ], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "", + "hide": 2, + "includeAll": false, + "multi": false, + "name": "cluster", + "options": [ ], + "query": { + "query": "label_values(certmanager_certificate_ready_status{}, cluster)", + "refId": "Prometheus-cluster-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Cert Manager / Overview", + "uid": "TvuRo2iMk", + "version": 3, + "weekStart": "" +} diff --git a/assets/cilium-enterprise/alerts.yaml b/assets/cilium-enterprise/alerts.yaml index 2384df9..0c811a7 100644 --- a/assets/cilium-enterprise/alerts.yaml +++ b/assets/cilium-enterprise/alerts.yaml @@ -19,7 +19,7 @@ groups: This may cause problems for Pod scheduling summary: API calls to Cilium Agent API to create or update Endpoints are failing. expr: sum(rate(cilium_k8s_client_api_calls_total{method=~"(PUT|POST|PATCH)", endpoint="endpoint",return_code!~"2[0-9][0-9]"}[5m])) - by (pod, method, return_code) + by (pod, method, return_code) > 0 for: 5m labels: severity: warning @@ -31,7 +31,7 @@ groups: summary: Cilium Endpoint API endpoint rate limiter is reporting errors while doing endpoint create. expr: sum(rate(cilium_api_limiter_processed_requests_total{api_call=~"endpoint-create", - outcome="fail"}[1m])) by (pod, api_call) + outcome="fail"}[1m])) by (pod, api_call) > 0 for: 5m labels: severity: info @@ -43,7 +43,7 @@ groups: This could indicate issues with Ciliums ability to create endpoints which can result in failure to schedule Kubernetes Pods. summary: API calls to Cilium Endpoints API are failing due to server errors. expr: sum(rate(cilium_agent_api_process_time_seconds_count{return_code=~"5[0-9][0-9]", - path="/v1/endpoint"}[5m])) by (pod, return_code) + path="/v1/endpoint"}[5m])) by (pod, return_code) > 0 for: 5m labels: severity: warning @@ -133,7 +133,7 @@ groups: error rate for response code: {{$labels.response_code}} on endpoint {{$labels.endpoint}}.' summary: Cilium Agent API on Pod is experiencing a high error rate. expr: sum(rate(cilium_k8s_client_api_calls_total{endpoint!="metrics",return_code!~"2[0-9][0-9]"}[5m])) - by (pod, endpoint, return_code) + by (pod, endpoint, return_code) > 0 for: 5m labels: severity: info @@ -224,3 +224,52 @@ groups: for: 15m labels: severity: info +- name: Cilium Clustermesh + rules: + - alert: CiliumAgentRemoteClusterNotReady + annotations: + description: Agent can't mesh with {{$labels.target_cluster}} + summary: Agent can't mesh with remote cluster. + expr: count(cilium_clustermesh_remote_cluster_readiness_status < 1) by (source_cluster, + target_cluster) > 0 + for: 5m + labels: + severity: critical + - alert: CiliumAgentRemoteClusterFailing + annotations: + description: Agent fails to mesh with {{$labels.target_cluster}} + summary: Agent fails to mesh with remote cluster. + expr: sum(rate(cilium_clustermesh_remote_cluster_failures[5m])) by (source_cluster, + target_cluster) > 0 + for: 5m + labels: + severity: critical +- name: Cilium Kvstoremesh + rules: + - alert: CiliumKvstoremeshRemoteClusterNotReady + annotations: + description: Kvstoremesh can't mesh with {{$labels.target_cluster}} + summary: Kvstoremesh can't mesh with remote cluster. + expr: count(cilium_kvstoremesh_remote_cluster_readiness_status < 1) by (source_cluster, + target_cluster) > 0 + for: 5m + labels: + severity: critical + - alert: CiliumKvstoremeshRemoteClusterFailing + annotations: + description: Kvstoremesh fails to mesh with {{$labels.target_cluster}} + summary: Kvstoremesh fails to mesh with remote cluster. + expr: sum(rate(cilium_kvstoremesh_remote_cluster_failures[5m])) by (source_cluster, + target_cluster) > 0 + for: 5m + labels: + severity: critical + - alert: CiliumKvstoremeshErrors + annotations: + description: Kvstoremesh fails to mesh with {{$labels.target_cluster}} + summary: Kvstoremesh fails to mesh with remote cluster. + expr: sum(rate(cilium_kvstoremesh_kvstore_sync_errors_total[5m])) by (source_cluster) + > 0 + for: 5m + labels: + severity: critical diff --git a/assets/clickhouse/dashboards/clickhouse-logs.json b/assets/clickhouse/dashboards/clickhouse-logs.json index 273e1d5..d8d8bfc 100644 --- a/assets/clickhouse/dashboards/clickhouse-logs.json +++ b/assets/clickhouse/dashboards/clickhouse-logs.json @@ -142,17 +142,19 @@ }, "gridPos": { "h": 6, - "w": 24 + "w": 24, + "x": 0, + "y": 0 }, "id": 1, - "interval": "30s", + "maxDataPoints": 100, "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -182,7 +184,9 @@ }, "gridPos": { "h": 18, - "w": 24 + "w": 24, + "x": 0, + "y": 18 }, "id": 2, "options": { @@ -192,7 +196,7 @@ "showTime": false, "wrapLogMessage": true }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -207,7 +211,7 @@ } ], "refresh": "1m", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "clickhouse-mixin" ], diff --git a/assets/consul/dashboards/consul-overview.json b/assets/consul/dashboards/consul-overview.json deleted file mode 100644 index 6881207..0000000 --- a/assets/consul/dashboards/consul-overview.json +++ /dev/null @@ -1,699 +0,0 @@ -{ - "annotations": { - "list": [ ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "100px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "colorBackground": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": "instance", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "consul_up{job=~\"$job\",instance=~\"$instance\"}", - "format": "time_series", - "instant": true, - "refId": "A" - } - ], - "thresholds": "0.5,0.5", - "timeFrom": null, - "timeShift": null, - "title": "$instance", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "valueMaps": [ - { - "op": "=", - "text": "DOWN", - "value": "0" - }, - { - "op": "=", - "text": "UP", - "value": "1" - } - ], - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Up", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "100px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "colorBackground": true, - "colors": [ - "rgba(237, 129, 40, 0.89)", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": "instance", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(rate(consul_raft_leader_lastcontact_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) > bool 0)\n or\n(consul_up{job=~\"$job\",instance=~\"$instance\"} == bool 0)\n", - "format": "time_series", - "instant": true, - "refId": "A" - } - ], - "thresholds": "0.5,0.5", - "timeFrom": null, - "timeShift": null, - "title": "$instance", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "valueMaps": [ - { - "op": "=", - "text": "FOLLOWER", - "value": "0" - }, - { - "op": "=", - "text": "LEADER", - "value": "1" - } - ], - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Leader", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "100px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "colorBackground": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": "instance", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "consul_raft_leader{job=~\"$job\",instance=~\"$instance\"}", - "format": "time_series", - "instant": true, - "refId": "A" - } - ], - "thresholds": "0.5,0.5", - "timeFrom": null, - "timeShift": null, - "title": "$instance", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "valueMaps": [ - { - "op": "=", - "text": "NO LEADER", - "value": "0" - }, - { - "op": "=", - "text": "HAS LEADER", - "value": "1" - } - ], - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Has Leader", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "100px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "colorBackground": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": "instance", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "consul_raft_peers{job=~\"$job\",instance=~\"$instance\"}", - "format": "time_series", - "instant": true, - "refId": "A" - } - ], - "thresholds": "1,2", - "timeFrom": null, - "timeShift": null, - "title": "$instance", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "# Peers", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(consul_http_request_count{job=~\"$job\"}[$__rate_interval])) by (instance)", - "format": "time_series", - "legendFormat": "{{instance}}", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(consul_http_request{job=~\"$job\", quantile=\"0.99\"}) by (instance)", - "format": "time_series", - "legendFormat": "99th Percentile", - "legendLink": "{{instance}}" - }, - { - "expr": "max(consul_http_request{job=~\"$job\", quantile=\"0.5\"}) by (instance)", - "format": "time_series", - "legendFormat": "50th Percentile", - "legendLink": "{{instance}}" - }, - { - "expr": "sum(rate(consul_http_request{job=~\"$job\"}[$__rate_interval])) by (instance) / sum(rate(consul_http_request{job=~\"$job\"}[$__rate_interval])) by (instance)", - "format": "time_series", - "legendFormat": "Average", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Consul Server", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ ], - "templating": { - "list": [ - { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data source", - "name": "datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": ".+", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": true, - "label": "job", - "multi": true, - "name": "job", - "options": [ ], - "query": "label_values(consul_up, job)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": true, - "label": "instance", - "multi": true, - "name": "instance", - "options": [ ], - "query": "label_values(consul_up{job=~\"$job\"}, instance)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Consul Overview", - "uid": "f71ad233ae18512edec190eee2558346", - "version": 0 -} diff --git a/assets/cortex/alerts.yaml b/assets/cortex/alerts.yaml index 27e9daf..bfb33b0 100644 --- a/assets/cortex/alerts.yaml +++ b/assets/cortex/alerts.yaml @@ -33,18 +33,6 @@ groups: for: 15m labels: severity: warning - - alert: CortexTableSyncFailure - annotations: - message: | - {{ $labels.job }} is experiencing {{ printf "%.2f" $value }}% errors syncing tables. - expr: | - 100 * rate(cortex_table_manager_sync_duration_seconds_count{status_code!~"2.."}[15m]) - / - rate(cortex_table_manager_sync_duration_seconds_count[15m]) - > 10 - for: 30m - labels: - severity: critical - alert: CortexQueriesIncorrect annotations: message: | @@ -113,26 +101,6 @@ groups: changes(process_start_time_seconds{job=~".+(cortex|ingester.*)"}[30m]) >= 2 labels: severity: warning - - alert: CortexTransferFailed - annotations: - message: | - {{ $labels.job }}/{{ $labels.instance }} transfer failed. - expr: | - max_over_time(cortex_shutdown_duration_seconds_count{op="transfer",status!="success"}[15m]) - for: 5m - labels: - severity: critical - - alert: CortexOldChunkInMemory - annotations: - message: | - {{ $labels.job }}/{{ $labels.instance }} has very old unflushed chunk in memory. - expr: | - (time() - cortex_oldest_unflushed_chunk_timestamp_seconds > 36000) - and - (cortex_oldest_unflushed_chunk_timestamp_seconds > 0) - for: 5m - labels: - severity: warning - alert: CortexKVStoreFailure annotations: message: | @@ -224,48 +192,6 @@ groups: for: 5m labels: severity: critical -- name: cortex_wal_alerts - rules: - - alert: CortexWALCorruption - annotations: - message: | - {{ $labels.job }}/{{ $labels.instance }} has a corrupted WAL or checkpoint. - expr: | - increase(cortex_ingester_wal_corruptions_total[5m]) > 0 - labels: - severity: critical - - alert: CortexCheckpointCreationFailed - annotations: - message: | - {{ $labels.job }}/{{ $labels.instance }} failed to create checkpoint. - expr: | - increase(cortex_ingester_checkpoint_creations_failed_total[10m]) > 0 - labels: - severity: warning - - alert: CortexCheckpointCreationFailed - annotations: - message: | - {{ $labels.job }}/{{ $labels.instance }} is failing to create checkpoint. - expr: | - increase(cortex_ingester_checkpoint_creations_failed_total[1h]) > 1 - labels: - severity: critical - - alert: CortexCheckpointDeletionFailed - annotations: - message: | - {{ $labels.job }}/{{ $labels.instance }} failed to delete checkpoint. - expr: | - increase(cortex_ingester_checkpoint_deletions_failed_total[10m]) > 0 - labels: - severity: warning - - alert: CortexCheckpointDeletionFailed - annotations: - message: | - {{ $labels.instance }} is failing to delete checkpoint. - expr: | - increase(cortex_ingester_checkpoint_deletions_failed_total[2h]) > 1 - labels: - severity: critical - name: cortex-rollout-alerts rules: - alert: CortexRolloutStuck @@ -314,29 +240,12 @@ groups: severity: warning - name: cortex-provisioning rules: - - alert: CortexProvisioningMemcachedTooSmall - annotations: - message: | - Chunk memcached cluster in {{ $labels.cluster }}/{{ $labels.namespace }} is too small, should be at least {{ printf "%.2f" $value }}GB. - expr: | - ( - 4 * - sum by (cluster, namespace) (cortex_ingester_memory_series * cortex_ingester_chunk_size_bytes_sum / cortex_ingester_chunk_size_bytes_count) - / 1e9 - ) - > - ( - sum by (cluster, namespace) (memcached_limit_bytes{job=~".+/memcached"}) / 1e9 - ) - for: 15m - labels: - severity: warning - alert: CortexProvisioningTooManyActiveSeries annotations: message: | The number of in-memory series per ingester in {{ $labels.cluster }}/{{ $labels.namespace }} is too high. expr: | - avg by (cluster, namespace) (cortex_ingester_memory_series) > 1.6e6 + avg by (cluster, namespace) (cortex_ingester_memory_series) > 3.2e6 for: 2h labels: severity: warning @@ -345,7 +254,7 @@ groups: message: | Ingesters in {{ $labels.cluster }}/{{ $labels.namespace }} ingest too many samples per second. expr: | - avg by (cluster, namespace) (rate(cortex_ingester_ingested_samples_total[1m])) > 80e3 + avg by (cluster, namespace) (rate(cortex_ingester_ingested_samples_total[1m])) > 160e3 for: 15m labels: severity: warning diff --git a/assets/cortex/dashboards/alertmanager.json b/assets/cortex/dashboards/alertmanager.json index 4065273..b1aec55 100644 --- a/assets/cortex/dashboards/alertmanager.json +++ b/assets/cortex/dashboards/alertmanager.json @@ -27,232 +27,151 @@ "height": "100px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cluster_job_pod:cortex_alertmanager_alerts:sum{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Total Alerts", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cluster_job_pod:cortex_alertmanager_silences:sum{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Total Silences", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "max(cortex_alertmanager_tenants_discovered{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Tenants", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" } ], "repeat": null, @@ -267,90 +186,59 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 4, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 12, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cluster_job:cortex_alertmanager_alerts_received_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})\n-\nsum(cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "APS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -365,253 +253,177 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})\n-\nsum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "NPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "(\nsum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}) by(integration)\n-\nsum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}) by(integration)\n) > 0\nor on () vector(0)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success - {{ integration }}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}) by(integration)", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed - {{ integration }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "NPS by integration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_alertmanager_notification_latency_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_alertmanager_notification_latency_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -650,157 +462,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_v1_alerts|alertmanager\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_v1_alerts|alertmanager\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 9, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_v1_alerts|alertmanager\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_v1_alerts|alertmanager\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_v1_alerts|alertmanager\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_v1_alerts|alertmanager\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -833,140 +745,103 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "rps" + }, + "overrides": [ ] + }, "fill": 10, "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Operations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 11, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Error rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -987,81 +862,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 12, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1082,81 +943,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 13, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1189,81 +1036,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 14, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1284,81 +1117,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 15, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1379,81 +1198,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 16, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1474,81 +1279,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 17, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1581,235 +1372,157 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "max by(pod) (cortex_alertmanager_tenants_owned{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod Tenants", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (cluster_job_pod:cortex_alertmanager_alerts:sum{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod Alerts", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (cluster_job_pod:cortex_alertmanager_silences:sum{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod Silences", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1824,244 +1537,155 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 21, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_alertmanager_sync_configs_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_sync_configs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_alertmanager_sync_configs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Syncs/sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 22, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(reason) (rate(cortex_alertmanager_sync_configs_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{reason}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Syncs/sec (By Reason)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 23, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum (rate(cortex_alertmanager_ring_check_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "errors", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Ring Check Errors/sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2076,158 +1700,115 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 24, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(outcome) (rate(cortex_alertmanager_state_initial_sync_completed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))", "format": "time_series", "interval": "1m", - "intervalFactor": 2, "legendFormat": "{{outcome}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Initial syncs /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 25, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))", "format": "time_series", "interval": "1m", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Initial sync duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2248,90 +1829,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 26, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))\n", "format": "time_series", "interval": "1m", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))", "format": "time_series", "interval": "1m", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Fetch state from other alertmanagers /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2346,262 +1896,169 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 27, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 27, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cluster_job:cortex_alertmanager_state_replication_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})\n-\nsum(cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Replicate state to other alertmanagers /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 28, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cluster_job:cortex_alertmanager_partial_state_merges_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})\n-\nsum(cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Merge state from other alertmanagers /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 29, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 29, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_alertmanager_state_persist_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_state_persist_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_alertmanager_state_persist_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/alertmanager\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Persist state to remote storage /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2625,7 +2082,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/cortex/dashboards/cortex-compactor-resources.json b/assets/cortex/dashboards/cortex-compactor-resources.json index 73ac8cb..d399b64 100644 --- a/assets/cortex/dashboards/cortex-compactor-resources.json +++ b/assets/cortex/dashboards/cortex-compactor-resources.json @@ -27,30 +27,40 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "seriesOverrides": [ { "alias": "limit", @@ -58,89 +68,64 @@ "fill": 0 } ], - "spaceLength": 10, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "seriesOverrides": [ { "alias": "limit", @@ -148,45 +133,28 @@ "fill": 0 } ], - "spaceLength": 10, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} > 0)", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "bytes", @@ -207,61 +175,55 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "bytes", @@ -294,63 +256,55 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, "fill": 10, "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Receive Bandwidth", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "Bps", @@ -371,63 +325,55 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, "fill": 10, "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Transmit Bandwidth", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "Bps", @@ -460,63 +406,55 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, "fill": 10, "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=\"compactor\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Writes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "Bps", @@ -537,63 +475,55 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, "fill": 10, "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=\"compactor\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Reads", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "Bps", @@ -614,63 +544,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 8, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=\"compactor\"\n }\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{persistentvolumeclaim}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Space Utilization", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -712,7 +631,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/cortex/dashboards/cortex-compactor.json b/assets/cortex/dashboards/cortex-compactor.json index 52385a7..f9b2871 100644 --- a/assets/cortex/dashboards/cortex-compactor.json +++ b/assets/cortex/dashboards/cortex-compactor.json @@ -33,158 +33,120 @@ "started": "#34CCEB" }, "bars": true, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Per-instance runs\nNumber of times a compactor instance triggers a compaction across all tenants that it manages.\n\n", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, "lines": false, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_compactor_runs_started_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "started", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_compactor_runs_completed_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "completed", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_compactor_runs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per-instance runs / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Tenants compaction progress\nIn a multi-tenant cluster, display the progress of tenants that are compacted while compaction is running.\nReset to 0 after the compaction run is completed for all tenants in the shard.\n\n", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "(\n cortex_compactor_tenants_processing_succeeded{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"} +\n cortex_compactor_tenants_processing_failed{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"} +\n cortex_compactor_tenants_skipped{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}\n) / cortex_compactor_tenants_discovered{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Tenants compaction progress", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -217,160 +179,117 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Compacted blocks / sec\nRate of blocks that are generated as a result of a compaction operation.\n\n", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(prometheus_tsdb_compactions_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "blocks", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Compacted blocks / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Per-block compaction duration\nDisplay the amount of time that it has taken to generate a single compacted block.\n\n", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 4, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(prometheus_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) * 1e3 / sum(rate(prometheus_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per-block compaction duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -403,159 +322,101 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "avg(max by(user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "avg", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Average blocks / tenant", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Tenants with largest number of blocks\nThe 10 tenants with the largest number of blocks.\n\n", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "topk(10, max by(user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{user}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Tenants with largest number of blocks", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -570,152 +431,114 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_compactor_blocks_marked_for_deletion_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "blocks", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Blocks marked for deletion / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { "aliasColors": { "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_compactor_blocks_cleaned_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_compactor_block_cleanup_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Blocks deletions / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ops", @@ -752,71 +575,62 @@ "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_compactor_meta_syncs_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) - sum(rate(cortex_compactor_meta_sync_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_compactor_meta_sync_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Metadata Syncs / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ops", @@ -837,81 +651,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 10, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_compactor_meta_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_compactor_meta_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Metadata Sync Duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -944,140 +744,103 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "rps" + }, + "overrides": [ ] + }, "fill": 10, "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Operations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 12, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Error rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -1098,81 +861,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 13, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1193,81 +942,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 14, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1300,81 +1035,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 15, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1395,81 +1116,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 16, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1490,81 +1197,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 17, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1585,81 +1278,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 18, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1701,7 +1380,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/cortex/dashboards/cortex-config.json b/assets/cortex/dashboards/cortex-config.json index 1a576f4..99dd8c2 100644 --- a/assets/cortex/dashboards/cortex-config.json +++ b/assets/cortex/dashboards/cortex-config.json @@ -27,81 +27,55 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "instances" + }, + "overrides": [ ] + }, "fill": 10, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 12, "stack": true, - "steppedLine": false, "targets": [ { "expr": "count(cortex_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}) by (sha256)", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "sha256:{{sha256}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Startup config file hashes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "instances", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -116,81 +90,55 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "instances" + }, + "overrides": [ ] + }, "fill": 10, "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 12, "stack": true, - "steppedLine": false, "targets": [ { "expr": "count(cortex_runtime_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}) by (sha256)", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "sha256:{{sha256}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Runtime config file hashes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "instances", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -214,7 +162,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/cortex/dashboards/cortex-object-store.json b/assets/cortex/dashboards/cortex-object-store.json index 99e9de7..b2f986e 100644 --- a/assets/cortex/dashboards/cortex-object-store.json +++ b/assets/cortex/dashboards/cortex-object-store.json @@ -27,158 +27,103 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "rps" + }, + "overrides": [ ] + }, "fill": 10, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(component) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{component}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "RPS / component", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(component) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by(component) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{component}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Error rate / component", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -193,158 +138,103 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "rps" + }, + "overrides": [ ] + }, "fill": 10, "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "RPS / operation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 4, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Error rate / operation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -359,81 +249,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -454,81 +330,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -549,81 +411,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -656,81 +504,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 8, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -751,81 +585,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 9, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -846,81 +666,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 10, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -962,7 +768,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/cortex/dashboards/cortex-queries.json b/assets/cortex/dashboards/cortex-queries.json index 352bc57..36ee8f7 100644 --- a/assets/cortex/dashboards/cortex-queries.json +++ b/assets/cortex/dashboards/cortex-queries.json @@ -27,81 +27,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_query_frontend_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_frontend_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Queue Duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -122,84 +108,70 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_query_frontend_retries_sum{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) * 1 / sum(rate(cortex_query_frontend_retries_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Retries", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -217,81 +189,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "cortex_query_frontend_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{cluster}} / {{namespace}} / {{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Queue Length", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -306,81 +249,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 4, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Queue Duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -401,81 +330,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{cluster}} / {{namespace}} / {{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Queue Length", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -490,141 +390,101 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Intervals per Query\nThe average number of splitted queries (partitioned by time) executed a single input query.\n\n", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_frontend_split_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[1m])) / sum(rate(cortex_frontend_query_range_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\", method=\"split_by_interval\"}[1m]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "splitting rate", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Intervals per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_cache_hits{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[1m])) / sum(rate(cortex_cache_fetched_keys{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[1m])) or\nsum(rate(cortex_cache_hits_total{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[1m])) / sum(rate(cortex_cache_fetched_keys_total{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[1m]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Hit Rate", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Results Cache Hit %", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -645,81 +505,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 8, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_cache_fetched_keys{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[1m])) - sum(rate(cortex_cache_hits{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[1m])) or\nsum(rate(cortex_cache_fetched_keys_total{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[1m])) - sum(rate(cortex_cache_hits_total{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[1m]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Miss Rate", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Results Cache misses", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -734,64 +565,53 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Sharded Queries Ratio\nThe % of queries that have been successfully rewritten and executed in a shardable way.\nThis panel takes in account only type of queries which are supported by query sharding (eg. range queries).\n\n", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 9, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_frontend_query_sharding_rewrites_succeeded_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) /\nsum(rate(cortex_frontend_query_sharding_rewrites_attempted_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "sharded queries ratio", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Sharded Queries Ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -812,85 +632,71 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Number of Sharded Queries per Query\nHow many sharded queries have been executed for a single input query. It tracks only queries which have\nbeen successfully rewritten in a shardable way.\n\n", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 10, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_frontend_sharded_queries_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval])) * 1 / sum(rate(cortex_frontend_sharded_queries_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Number of Sharded Queries per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -920,63 +726,223 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "fill": 10, "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{slice}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Stages", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "id": 12, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, + "span": 4, + "targets": [ + { + "expr": "sum(rate(cortex_cache_fetched_keys{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\",name=\"chunksmemcache\"}[1m])) - sum(rate(cortex_cache_hits{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\",name=\"chunksmemcache\"}[1m])) or\nsum(rate(cortex_cache_fetched_keys_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\",name=\"chunksmemcache\"}[1m])) - sum(rate(cortex_cache_hits_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\",name=\"chunksmemcache\"}[1m]))\n", + "format": "time_series", + "interval": "15s", + "legendFormat": "Hit rate", + "legendLink": null + } + ], + "title": "Chunk cache misses", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "id": 13, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum(rate(cortex_cache_corrupt_chunks_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[1m]))", + "format": "time_series", + "interval": "15s", + "legendFormat": "Corrupt chunks", + "legendLink": null + } + ], + "title": "Chunk cache corruptions", + "type": "timeseries" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Querier", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "id": 14, + "links": [ ], + "nullPointMode": "null as zero", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_series_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})) * 1", + "format": "time_series", + "legendFormat": "99th percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_series_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})) * 1", + "format": "time_series", + "legendFormat": "50th percentile", + "refId": "B" + }, + { + "expr": "1 * sum(cluster_job:cortex_ingester_queried_series_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}) / sum(cluster_job:cortex_ingester_queried_series_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})", + "format": "time_series", + "legendFormat": "Average", + "refId": "C" + } + ], + "title": "Series per Query", + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -997,339 +963,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(cortex_cache_fetched_keys{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\",name=\"chunksmemcache\"}[1m])) - sum(rate(cortex_cache_hits{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\",name=\"chunksmemcache\"}[1m])) or\nsum(rate(cortex_cache_fetched_keys_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\",name=\"chunksmemcache\"}[1m])) - sum(rate(cortex_cache_hits_total{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\",name=\"chunksmemcache\"}[1m]))\n", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "Hit rate", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Chunk cache misses", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "overrides": [ ] }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(cortex_cache_corrupt_chunks_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[1m]))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "Corrupt chunks", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Chunk cache corruptions", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Querier", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_series_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_series_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 - }, - { - "expr": "1 * sum(cluster_job:cortex_ingester_queried_series_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}) / sum(cluster_job:cortex_ingester_queried_series_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Series per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_chunks_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})) * 1", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_chunks_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})) * 1", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1 * sum(cluster_job:cortex_ingester_queried_chunks_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}) / sum(cluster_job:cortex_ingester_queried_chunks_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Chunks per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1347,81 +1041,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 16, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_samples_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})) * 1", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_samples_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})) * 1", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1 * sum(cluster_job:cortex_ingester_queried_samples_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}) / sum(cluster_job:cortex_ingester_queried_samples_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Samples per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1451,84 +1131,70 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 17, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Number of store-gateways hit per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1546,84 +1212,70 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 18, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Refetches of missing blocks per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1641,63 +1293,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 19, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[1m])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[1m]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Failure Rate", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Consistency checks failed", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -1730,265 +1371,191 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 20, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "max(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Max", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Min", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "avg(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Bucket indexes loaded (per querier)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { "aliasColors": { "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_bucket_index_loads_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_bucket_index_load_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_bucket_index_load_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Bucket indexes load / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 22, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_bucket_index_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_index_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Bucket indexes load latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2021,235 +1588,154 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 23, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_bucket_store_series_blocks_queried_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "blocks", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Blocks queried / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(data_type) (rate(cortex_bucket_store_series_data_fetched_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{data_type}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Data fetched / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(data_type) (rate(cortex_bucket_store_series_data_touched_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{data_type}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Data touched / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2264,81 +1750,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 26, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_series_get_all_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_series_get_all_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_bucket_store_series_get_all_duration_seconds_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_series_get_all_duration_seconds_count{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Series fetch duration (per request)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2359,81 +1831,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 27, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 27, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_series_merge_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_series_merge_duration_seconds_bucket{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_bucket_store_series_merge_duration_seconds_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_series_merge_duration_seconds_count{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Series merge duration (per request)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2454,81 +1912,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 28, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_bucket_store_series_result_series_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) / sum(rate(cortex_bucket_store_series_result_series_count{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "avg series returned", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Series returned (per request)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2543,259 +1972,176 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 29, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 29, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "cortex_bucket_store_blocks_loaded{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}", + "expr": "sum(cortex_bucket_store_blocks_loaded{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}) without (user)", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Blocks currently loaded", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { "aliasColors": { "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 30, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_bucket_store_block_loads_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Blocks loaded / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { "aliasColors": { "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 31, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_bucket_store_block_drops_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Blocks dropped / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2810,158 +2156,115 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 32, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 32, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "cortex_bucket_store_indexheader_lazy_load_total{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"} - cortex_bucket_store_indexheader_lazy_unload_total{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Lazy loaded index-headers", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 33, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 33, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Index-header lazy load duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2982,63 +2285,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 34, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 34, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_bucket_store_series_hash_cache_hits_total{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))\n/\nsum(rate(cortex_bucket_store_series_hash_cache_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"}[$__rate_interval]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "hit ratio", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Series hash cache hit ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -3080,7 +2372,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/cortex/dashboards/cortex-reads.json b/assets/cortex/dashboards/cortex-reads.json index 1ee1524..4a922b2 100644 --- a/assets/cortex/dashboards/cortex-reads.json +++ b/assets/cortex/dashboards/cortex-reads.json @@ -50,158 +50,104 @@ "height": "100px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Instant queries per second\nRate of instant queries per second being made to the system.\nIncludes both queries made to the /prometheus API as\nwell as queries from the ruler.\n\n", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "reqps", "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(\n rate(\n cortex_request_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\",\n route=~\"(prometheus|api_prom)_api_v1_query\"\n }[$__rate_interval]\n )\n) +\nsum(\n rate(\n cortex_prometheus_rule_evaluations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Instant queries / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Range queries per second\nRate of range queries per second being made to\nCortex via the /prometheus API.\n\n", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "reqps", "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(\n rate(\n cortex_request_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\",\n route=~\"(prometheus|api_prom)_api_v1_query_range\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Range queries / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" } ], "repeat": null, @@ -222,157 +168,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -393,73 +439,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 6, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -480,157 +512,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 8, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -651,73 +783,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 9, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -749,160 +867,260 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 12, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency (Time in Queue)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -941,157 +1159,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_cache_request_duration_seconds_count{method=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_cache_request_duration_seconds_count{method=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 14, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_method:cortex_cache_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\", method=~\"frontend.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_method:cortex_cache_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\", method=~\"frontend.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_method:cortex_cache_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\", method=~\"frontend.+\"}) / sum(cluster_job_method:cortex_cache_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(query-frontend.*|cortex$)\", method=~\"frontend.+\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1130,157 +1448,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 16, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1301,73 +1719,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 17, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1388,157 +1792,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 19, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1559,73 +2063,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 20, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1646,157 +2136,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 22, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\", route=~\"/gatewaypb.StoreGateway/.*\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\", route=~\"/gatewaypb.StoreGateway/.*\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\", route=~\"/gatewaypb.StoreGateway/.*\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\", route=~\"/gatewaypb.StoreGateway/.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1817,73 +2407,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 23, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\", route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1898,158 +2474,118 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n component=\"store-gateway\",\n name=\"index-cache\",\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 25, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency (getmulti)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2070,64 +2606,53 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Hit Ratio\nEven if you do not set up memcached for the blocks index cache, you will still see data in this panel because Cortex by default has an\nin-memory blocks index cache.\n\n", - "fill": 1, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 26, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(item_type) (\n rate(\n thanos_store_index_cache_hits_total{\n component=\"store-gateway\",\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"\n }[$__rate_interval]\n )\n)\n/\nsum by(item_type) (\n rate(\n thanos_store_index_cache_requests_total{\n component=\"store-gateway\",\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{item_type}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Hit ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -2160,63 +2685,55 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 27, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ops", @@ -2237,81 +2754,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 28, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency (getmulti)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2332,63 +2835,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 29, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 29, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "items", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Hit ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -2421,63 +2913,55 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 30, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ops", @@ -2498,81 +2982,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 31, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 31, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency (getmulti)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2593,63 +3063,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 32, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 32, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(store-gateway|cortex$)\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "items", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Hit ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -2682,63 +3141,55 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 33, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ops", @@ -2759,81 +3210,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 34, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 34, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency (getmulti)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2854,63 +3291,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 35, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 35, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/(querier.*|cortex$)\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "items", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Hit ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -2943,140 +3369,103 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "rps" + }, + "overrides": [ ] + }, "fill": 10, "id": 36, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Operations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 37, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 37, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Error rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -3097,81 +3486,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 38, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 38, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -3192,81 +3567,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 39, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 39, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -3299,81 +3660,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 40, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 40, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -3394,81 +3741,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 41, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 41, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -3489,81 +3822,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 42, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 42, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -3584,81 +3903,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 43, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 43, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -3691,140 +3996,103 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "rps" + }, + "overrides": [ ] + }, "fill": 10, "id": 44, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Operations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 45, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 45, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Error rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -3845,81 +4113,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 46, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 46, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -3940,81 +4194,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 47, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 47, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -4047,81 +4287,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 48, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 48, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -4142,81 +4368,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 49, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 49, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -4237,81 +4449,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 50, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 50, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -4332,81 +4530,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 51, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 51, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -4448,7 +4632,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/cortex/dashboards/cortex-rollout-progress.json b/assets/cortex/dashboards/cortex-rollout-progress.json index b14eadb..a010a37 100644 --- a/assets/cortex/dashboards/cortex-rollout-progress.json +++ b/assets/cortex/dashboards/cortex-rollout-progress.json @@ -22,10 +22,6 @@ ], "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -55,7 +51,6 @@ "unit": "percentunit" } }, - "fill": 1, "gridPos": { "h": 8, "w": 10, @@ -63,19 +58,7 @@ "y": 0 }, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", "options": { "displayMode": "basic", "orientation": "horizontal", @@ -87,15 +70,6 @@ "values": false } }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "(\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\",statefulset=~\"cortex-gw|distributor|ingester.*|query-frontend.*|query-scheduler.*|querier.*|compactor|store-gateway|ruler|alertmanager\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n", @@ -114,47 +88,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Rollout progress", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "bargauge", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "bargauge" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -176,7 +113,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 2, @@ -184,28 +120,16 @@ "y": 0 }, "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]))\n", @@ -218,47 +142,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Writes - 2xx", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -288,7 +175,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 2, @@ -296,28 +182,16 @@ "y": 0 }, "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]))\n", @@ -330,47 +204,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Writes - 4xx", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -396,7 +233,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 2, @@ -404,28 +240,16 @@ "y": 0 }, "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]))\n", @@ -438,47 +262,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Writes - 5xx", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -508,7 +295,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 8, @@ -516,28 +302,16 @@ "y": 0 }, "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}))\n", @@ -550,47 +324,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Writes 99th Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "title": "Writes 99th latency", + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -612,7 +349,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 2, @@ -620,28 +356,16 @@ "y": 4 }, "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n", @@ -654,47 +378,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Reads - 2xx", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -724,7 +411,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 2, @@ -732,28 +418,16 @@ "y": 4 }, "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n", @@ -766,47 +440,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Reads - 4xx", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -832,7 +469,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 2, @@ -840,28 +476,16 @@ "y": 4 }, "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n", @@ -874,47 +498,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Reads - 5xx", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -944,7 +531,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 8, @@ -952,28 +538,16 @@ "y": 4 }, "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}))\n", @@ -986,47 +560,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Reads 99th Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "title": "Reads 99th latency", + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -1056,7 +593,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 8, "w": 10, @@ -1064,34 +600,13 @@ "y": 8 }, "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", "options": { "text": { "titleSize": 14, "valueSize": 14 } }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "kube_deployment_status_replicas_unavailable{cluster=~\"$cluster\", namespace=~\"$namespace\", deployment=~\"cortex-gw|distributor|ingester.*|query-frontend.*|query-scheduler.*|querier.*|compactor|store-gateway|ruler|alertmanager\"}\n> 0\n", @@ -1114,41 +629,8 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Unhealthy pods", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "stat" }, { "datasource": "$datasource", @@ -1214,12 +696,29 @@ "type": "table" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "gridPos": { "h": 8, "w": 8, @@ -1227,65 +726,34 @@ "y": 8 }, "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}))[1h:])\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "writes", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"(prometheus|api_prom)_api_v1_.+\"}))[1h:])\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "reads", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency vs 24h ago", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -1321,7 +789,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/cortex/dashboards/cortex-scaling.json b/assets/cortex/dashboards/cortex-scaling.json index 1a32463..2335656 100644 --- a/assets/cortex/dashboards/cortex-scaling.json +++ b/assets/cortex/dashboards/cortex-scaling.json @@ -49,195 +49,142 @@ "height": "400px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cluster" + }, + "properties": [ + { + "id": "displayName", + "value": "Cluster" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "namespace" + }, + "properties": [ + { + "id": "displayName", + "value": "Namespace" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deployment" + }, + "properties": [ + { + "id": "displayName", + "value": "Service" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reason" + }, + "properties": [ + { + "id": "displayName", + "value": "Reason" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Required Replicas" + } + ] + } + ] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "sort": { "col": 0, "desc": false }, - "spaceLength": 10, "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Required Replicas", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value", - "thresholds": [ ], - "type": "number", - "unit": "short" - }, - { - "alias": "Cluster", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "__name__", - "thresholds": [ ], - "type": "hidden", - "unit": "short" - }, - { - "alias": "Cluster", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "cluster", - "thresholds": [ ], - "type": "number", - "unit": "short" - }, - { - "alias": "Service", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "deployment", - "thresholds": [ ], - "type": "number", - "unit": "short" - }, - { - "alias": "Namespace", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "namespace", - "thresholds": [ ], - "type": "number", - "unit": "short" - }, - { - "alias": "Reason", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "reason", - "thresholds": [ ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ ], - "type": "string", - "unit": "short" - } - ], "targets": [ { "expr": "sort_desc(\n cluster_namespace_deployment_reason:required_replicas:count{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n > ignoring(reason) group_left\n cluster_namespace_deployment:actual_replicas:count{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\n", "format": "table", "instant": true, - "intervalFactor": 2, "legendFormat": "", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Workload-based scaling", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "table" } ], "repeat": null, @@ -261,7 +208,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/cortex/dashboards/cortex-slow-queries.json b/assets/cortex/dashboards/cortex-slow-queries.json index 67b4e80..c981ec5 100644 --- a/assets/cortex/dashboards/cortex-slow-queries.json +++ b/assets/cortex/dashboards/cortex-slow-queries.json @@ -171,7 +171,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/cortex/dashboards/cortex-writes.json b/assets/cortex/dashboards/cortex-writes.json index a8ea461..6c1b532 100644 --- a/assets/cortex/dashboards/cortex-writes.json +++ b/assets/cortex/dashboards/cortex-writes.json @@ -50,308 +50,200 @@ "height": "100px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cluster_namespace_job:cortex_distributor_received_samples:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Samples / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cortex_ingester_memory_series{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}\n/ on(cluster, namespace) group_left\nmax by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\"}))\n", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Active Series", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "count(count by(user) (cortex_ingester_active_series{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}))", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Tenants", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "reqps", "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}[5m]))", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" } ], "repeat": null, @@ -372,157 +264,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -543,73 +535,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 8, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_(v1|prom)_push\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -630,157 +608,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 10, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -801,73 +879,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 11, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -888,157 +952,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 13, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job:cortex_kv_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\"}) / sum(cluster_job:cortex_kv_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(distributor|cortex$)\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1077,157 +1241,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",route=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",route=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 15, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\", route=\"/cortex.Ingester/Push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\", route=\"/cortex.Ingester/Push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\", route=\"/cortex.Ingester/Push\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\", route=\"/cortex.Ingester/Push\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1248,73 +1512,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 16, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\", route=\"/cortex.Ingester/Push\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1335,157 +1585,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 18, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job:cortex_kv_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}) / sum(cluster_job:cortex_kv_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1522,168 +1872,127 @@ "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Uploaded blocks / sec\nThe rate of blocks being uploaded from the ingesters\nto object storage.\n\n", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_shipper_uploads_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_ingester_shipper_upload_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_ingester_shipper_upload_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Uploaded blocks / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Upload latency\nThe average, median (50th percentile), and 99th percentile time\nthe ingesters take to upload blocks to object storage.\n\n", - "fill": 1, - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 20, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\",component=\"ingester\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Upload latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1720,168 +2029,127 @@ "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Compactions per second\nIngesters maintain a local TSDB per-tenant on disk. Each TSDB maintains a head block for each\nactive time series; these blocks get periodically compacted (by default, every 2h).\nThis panel shows the rate of compaction operations across all TSDBs on all ingesters.\n\n", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_tsdb_compactions_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_ingester_tsdb_compactions_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Compactions / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Compaction latency\nThe average, median (50th percentile), and 99th percentile time ingesters take to compact TSDB head blocks\non the local filesystem.\n\n", - "fill": 1, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 22, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Compactions latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1918,329 +2186,237 @@ "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### WAL truncations per second\nThe WAL is truncated each time a new TSDB block is written. This panel measures the rate of\ntruncations.\n\n", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "WAL truncations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { "aliasColors": { "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Checkpoints created per second\nCheckpoints are created as part of the WAL truncation process.\nThis metric measures the rate of checkpoint creation.\n\n", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Checkpoints created / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### WAL truncations latency (including checkpointing)\nAverage time taken to perform a full WAL truncation,\nincluding the time taken for the checkpointing to complete.\n\n", - "fill": 1, - "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 25, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval])) / sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "avg", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "WAL truncations latency (includes checkpointing)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { "aliasColors": { "WAL": "#E24D42", "mmap-ed chunks": "#E28A42" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_wal_corruptions_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "WAL", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_ingester_tsdb_mmap_chunk_corruptions_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "mmap-ed chunks", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Corruptions / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ops", @@ -2282,7 +2458,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/cortex/dashboards/ruler.json b/assets/cortex/dashboards/ruler.json index c244200..bb9b2fd 100644 --- a/assets/cortex/dashboards/ruler.json +++ b/assets/cortex/dashboards/ruler.json @@ -27,308 +27,200 @@ "height": "100px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cortex_ruler_managers_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Active Configurations", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Total Rules", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "reqps", "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\", operation=\"/cortex.Ingester/QueryStream\"}[5m]))", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Read from Ingesters - QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "reqps", "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\", operation=\"/cortex.Ingester/Push\"}[5m]))", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Write to Ingesters - QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" } ], "repeat": null, @@ -343,167 +235,107 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval]))\n-\nsum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "EPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval]))\n /\nsum (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "average", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -524,157 +356,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 8, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -695,81 +627,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 9, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (route, le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/(gateway|cortex-gw|cortex-gw-internal)\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ route }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Per route p99 Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "title": "Per route p99 latency", + "type": "timeseries" } ], "repeat": null, @@ -790,160 +693,260 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 11, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -982,160 +985,260 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 13, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1168,84 +1271,70 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 14, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Number of store-gateways hit per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1263,84 +1352,70 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 15, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Refetches of missing blocks per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1358,63 +1433,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 16, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[1m])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[1m]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Failure Rate", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Consistency checks failed", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -1447,235 +1511,148 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 17, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(user) (rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval]))\n> 0\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ user }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Delivery Errors", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 18, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(user) (rate(cortex_prometheus_notifications_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_queue_capacity{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) > 0\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ user }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Queue Length", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 19, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by (user) (increase(cortex_prometheus_notifications_dropped_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) > 0\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ user }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Dropped", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1690,235 +1667,148 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 20, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(user) (rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) > 0", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ user }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Missed Iterations", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 21, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "rate(cortex_prometheus_rule_group_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])\n /\nrate(cortex_prometheus_rule_group_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ user }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 22, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(rule_group) (rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval])) > 0", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ rule_group }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Failures", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1933,81 +1823,52 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 23, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 12, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler|cortex$)\"}[$__rate_interval]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ user }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2022,140 +1883,103 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "rps" + }, + "overrides": [ ] + }, "fill": 10, "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Operations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 25, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Error rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -2176,81 +2000,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 26, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2271,81 +2081,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 27, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 27, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2378,81 +2174,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 28, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2473,81 +2255,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 29, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 29, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2568,81 +2336,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 30, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 30, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2663,81 +2417,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 31, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 31, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2779,7 +2519,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/cortex/rules.yaml b/assets/cortex/rules.yaml index 921cbff..43a95bc 100644 --- a/assets/cortex/rules.yaml +++ b/assets/cortex/rules.yaml @@ -501,7 +501,7 @@ groups: cortex_ingester_memory_series )[24h:] ) - / 1500000 + / 3000000 ) labels: deployment: ingester @@ -510,7 +510,7 @@ groups: - expr: | ceil( sum by (cluster, namespace) (cortex_overrides{limit_name="max_global_series_per_user"}) - * 3 * 0.59999999999999998 / 1500000 + * 3 * 0.59999999999999998 / 3000000 ) labels: deployment: ingester diff --git a/assets/docker/alerts.yaml b/assets/docker/alerts.yaml index 19765bd..0967ef4 100644 --- a/assets/docker/alerts.yaml +++ b/assets/docker/alerts.yaml @@ -1 +1 @@ -null +{} diff --git a/assets/docker/dashboards/docker-logs.json b/assets/docker/dashboards/docker-logs.json deleted file mode 100644 index 16de9e9..0000000 --- a/assets/docker/dashboards/docker-logs.json +++ /dev/null @@ -1,754 +0,0 @@ -{ - "__inputs": [ ], - "__requires": [ ], - "annotations": { - "list": [ ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [ - { - "asDropdown": false, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "docker" - ], - "targetBlank": false, - "title": "Docker Dashboards", - "type": "dashboards", - "url": "" - } - ], - "panels": [ - { - "collapse": false, - "collapsed": false, - "gridPos": { - "h": 0, - "w": 0, - "x": 0, - "y": 2 - }, - "id": 2, - "panels": [ ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Overview", - "titleSize": "h6", - "type": "row" - }, - { - "datasource": "$loki_datasource", - "description": "Total number of log lines including errors and warnings.", - "fieldConfig": { - "defaults": { - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(192, 216, 255)", - "value": 0 - } - ] - }, - "unit": "short" - } - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 2 - }, - "id": 3, - "links": [ ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7", - "targets": [ - { - "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", container=~\"$container\"}[$__interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Log Lines", - "transparent": false, - "type": "stat" - }, - { - "datasource": "$loki_datasource", - "description": "Total number of log lines of level: warning.", - "fieldConfig": { - "defaults": { - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(255, 152, 48)", - "value": 0 - } - ] - }, - "unit": "short" - } - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 2 - }, - "id": 4, - "links": [ ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7", - "targets": [ - { - "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", container=~\"$container\"} |= \"Warning\" [$__interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Warnings", - "transparent": false, - "type": "stat" - }, - { - "datasource": "$loki_datasource", - "description": "Total number of log lines of level: error.", - "fieldConfig": { - "defaults": { - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(242, 73, 92)", - "value": 0 - } - ] - }, - "unit": "short" - } - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 2 - }, - "id": 5, - "links": [ ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7", - "targets": [ - { - "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", container=~\"$container\"} |= \"Error\" [$__interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Errors", - "transparent": false, - "type": "stat" - }, - { - "datasource": "$loki_datasource", - "description": "Percentage of log lines with level: Error out of total log lines.", - "fieldConfig": { - "defaults": { - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(255, 166, 176)", - "value": 0 - }, - { - "color": "rgb(255, 115, 131)", - "value": 25 - }, - { - "color": "rgb(196, 22, 42)", - "value": 50 - } - ] - }, - "unit": "percent" - } - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 2 - }, - "id": 6, - "links": [ ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7", - "targets": [ - { - "expr": "sum( count_over_time({job=~\"$job\", instance=~\"$instance\", container=~\"$container\"} |= \"Error\" [$__interval]) ) / sum( count_over_time({job=~\"$job\", instance=~\"$instance\", container=~\"$container\"} [$__interval]) )", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Error Percentage", - "transparent": false, - "type": "stat" - }, - { - "datasource": "$loki_datasource", - "description": "Total number of bytes for log lines including errors and warnings.", - "fieldConfig": { - "defaults": { - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(184, 119, 217)", - "value": 0 - } - ] - }, - "unit": "bytes" - } - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 2 - }, - "id": 7, - "links": [ ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7", - "targets": [ - { - "expr": "sum(bytes_over_time({job=~\"$job\", instance=~\"$instance\", container=~\"$container\"} [$__interval]))", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Bytes Used", - "transparent": false, - "type": "stat" - }, - { - "datasource": { - "type": "loki", - "uid": "${loki_datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "fillOpacity": 50, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "A" - }, - "properties": [ - { - "id": "displayName", - "value": "Lines" - }, - { - "id": "color", - "value": { - "fixedColor": "super-light-blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "displayName", - "value": "Warnings" - }, - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "C" - }, - "properties": [ - { - "id": "displayName", - "value": "Errors" - }, - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 8, - "interval": "10s", - "maxDataPoints": 25, - "options": { - "barRadius": 0.25, - "barWidth": 0.69999999999999996, - "groupWidth": 0.5, - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom" - }, - "orientation": "auto", - "showValue": "never", - "stacking": "none", - "tooltip": { - "mode": "multi", - "sort": "none" - }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 100 - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "${loki_datasource}" - }, - "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", container=~\"$container\"}[$__interval]))", - "refId": "A" - }, - { - "datasource": { - "type": "loki", - "uid": "${loki_datasource}" - }, - "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", container=~\"$container\"} |= \"Warning\" [$__interval]))", - "hide": false, - "refId": "B" - }, - { - "datasource": { - "type": "loki", - "uid": "${loki_datasource}" - }, - "expr": "sum(count_over_time({job=~\"$job\", instance=~\"$instance\", container=~\"$container\"} |= \"Error\" [$__interval]))", - "hide": false, - "refId": "C" - } - ], - "title": "Historical Logs / Warnings / Errors", - "type": "barchart" - }, - { - "collapse": true, - "collapsed": true, - "gridPos": { - "h": 0, - "w": 0, - "x": 0, - "y": 12 - }, - "id": 9, - "panels": [ - { - "datasource": "$loki_datasource", - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 10, - "options": { - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "span": 12, - "targets": [ - { - "expr": "{job=~\"$job\", instance=~\"$instance\", container=~\"$container\"} |= \"Error\"", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Errors", - "type": "logs" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Errors", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": true, - "collapsed": true, - "gridPos": { - "h": 0, - "w": 0, - "x": 0, - "y": 20 - }, - "id": 11, - "panels": [ - { - "datasource": "$loki_datasource", - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 12, - "options": { - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "span": 12, - "targets": [ - { - "expr": "{job=~\"$job\", instance=~\"$instance\", container=~\"$container\"} |= \"Warning\"", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Warnings", - "type": "logs" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Warnings", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": true, - "collapsed": true, - "gridPos": { - "h": 0, - "w": 0, - "x": 0, - "y": 28 - }, - "id": 13, - "panels": [ - { - "datasource": "$loki_datasource", - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 14, - "options": { - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "span": 12, - "targets": [ - { - "expr": "{job=~\"$job\", instance=~\"$instance\", container=~\"$container\"}", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Full Log File", - "type": "logs" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Complete Log File", - "titleSize": "h6", - "type": "row" - } - ], - "refresh": "1m", - "rows": [ ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "docker" - ], - "templating": { - "list": [ - { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Prometheus Data Source", - "name": "prometheus_datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Loki Data Source", - "name": "loki_datasource", - "options": [ ], - "query": "loki", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": ".+", - "current": { }, - "datasource": "$prometheus_datasource", - "hide": 0, - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "options": [ ], - "query": "label_values(machine_scrape_error, job)", - "refresh": 1, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { }, - "datasource": "$prometheus_datasource", - "hide": 0, - "includeAll": true, - "label": "Instance", - "multi": true, - "name": "instance", - "options": [ ], - "query": "label_values(machine_scrape_error{job=~\"$job\"}, instance)", - "refresh": 1, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { }, - "datasource": "$prometheus_datasource", - "hide": 0, - "includeAll": true, - "label": "Container", - "multi": true, - "name": "container", - "options": [ ], - "query": "label_values(container_last_seen{job=~\"$job\", instance=~\"$instance\"}, name)", - "refresh": 1, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "default", - "title": "Docker Logs", - "uid": "integration-docker-logs", - "version": 0 -} diff --git a/assets/docker/dashboards/docker.json b/assets/docker/dashboards/docker.json deleted file mode 100644 index cae6a59..0000000 --- a/assets/docker/dashboards/docker.json +++ /dev/null @@ -1,1277 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 3, - "links": [ - { - "asDropdown": false, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "docker-integration" - ], - "targetBlank": false, - "title": "Docker Dashboards", - "type": "dashboards", - "url": "" - } - ], - "liveNow": false, - "panels": [ - { - "collapse": false, - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "$prometheus_datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 2, - "panels": [ ], - "showTitle": true, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$prometheus_datasource" - }, - "refId": "A" - } - ], - "title": "Overview", - "titleSize": "h6", - "type": "row" - }, - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "description": "Total number of running containers last seen by the exporter.", - "fieldConfig": { - "defaults": { - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 0, - "y": 4 - }, - "id": 3, - "links": [ ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "expr": "count(container_last_seen{job=~\"$job\", instance=~\"$instance\", name=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Containers", - "type": "stat" - }, - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "description": "Total number of distinct images found across running containers", - "fieldConfig": { - "defaults": { - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 4, - "y": 4 - }, - "id": 4, - "links": [ ], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "expr": "count (sum by (image) (container_last_seen{job=~\"$job\", instance=~\"$instance\", name=~\"$container\", image=~\".+\"}))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Images", - "type": "stat" - }, - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "description": "Cumulative cpu time consumed in seconds.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0.80000000000000004 - }, - { - "color": "#d44a3a", - "value": 0.90000000000000002 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 8, - "y": 4 - }, - "id": 5, - "links": [ ], - "maxDataPoints": 100, - "options": { - "minVizHeight": 200, - "minVizWidth": 200, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "expr": "sum (rate(container_cpu_usage_seconds_total{job=~\"$job\", instance=~\"$instance\", name=~\"$container\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "title": "CPU Utilization by Containers", - "type": "gauge" - }, - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "description": "Memory reserved by the containers on the machine.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0.80000000000000004 - }, - { - "color": "#d44a3a", - "value": 0.90000000000000002 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 12, - "y": 4 - }, - "id": 6, - "links": [ ], - "maxDataPoints": 100, - "options": { - "minVizHeight": 200, - "minVizWidth": 200, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "expr": "sum(container_spec_memory_reservation_limit_bytes{job=~\"$job\", instance=~\"$instance\", name=~\"$container\"}) / avg(machine_memory_bytes{job=~\"$job\", instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory Reserved by Containers", - "type": "gauge" - }, - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "description": "Current memory usage in bytes, including all memory regardless of when it was accessed.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0.80000000000000004 - }, - { - "color": "#d44a3a", - "value": 0.90000000000000002 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 16, - "y": 4 - }, - "id": 7, - "links": [ ], - "maxDataPoints": 100, - "options": { - "minVizHeight": 200, - "minVizWidth": 200, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "expr": "sum(container_memory_usage_bytes{job=~\"$job\", instance=~\"$instance\", name=~\"$container\"}) / avg(machine_memory_bytes{job=~\"$job\", instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory Utilization by Containers", - "type": "gauge" - }, - { - "collapse": false, - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "$prometheus_datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 8, - "panels": [ ], - "showTitle": true, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$prometheus_datasource" - }, - "refId": "A" - } - ], - "title": "Compute", - "titleSize": "h6", - "type": "row" - }, - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "description": "Cpu time consumed in seconds by container.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 0, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 9, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "expr": "avg by (name) (rate(container_cpu_usage_seconds_total{job=~\"$job\", instance=~\"$instance\", name=~\"$container\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A" - } - ], - "title": "CPU", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "description": "Current memory usage in bytes, including all memory regardless of when it was accessed by container.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 0, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 10, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "expr": "sum by (name) (container_memory_usage_bytes{job=~\"$job\", instance=~\"$instance\", name=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "collapse": false, - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "$prometheus_datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 11, - "panels": [ ], - "showTitle": true, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$prometheus_datasource" - }, - "refId": "A" - } - ], - "title": "Network", - "titleSize": "h6", - "type": "row" - }, - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "description": "Cumulative count of bytes transmitted.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 0, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*tx/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 21 - }, - "id": 12, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "expr": "sum by (name) (rate(container_network_receive_bytes_total{job=~\"$job\", instance=~\"$instance\", name=~\"$container\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}} rx", - "refId": "A" - }, - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "expr": "sum by (name) (rate(container_network_transmit_bytes_total{job=~\"$job\", instance=~\"$instance\", name=~\"$container\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{name}} tx", - "refId": "B" - } - ], - "title": "Bandwidth", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "description": "TCP sockets on containers by state.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 21 - }, - "id": 13, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "expr": "sum(container_network_tcp_usage_total{job=~\"$job\", instance=~\"$instance\", name=~\"$container\"}) by (tcp_state) > 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{tcp_state}}", - "refId": "A" - } - ], - "title": "TCP Sockets By State", - "type": "timeseries" - }, - { - "collapse": false, - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "$prometheus_datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 29 - }, - "id": 14, - "panels": [ ], - "showTitle": true, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$prometheus_datasource" - }, - "refId": "A" - } - ], - "title": "Storage", - "titleSize": "h6", - "type": "row" - }, - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "decimals": 2, - "displayName": "", - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "displayName", - "value": "Time" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "displayName", - "value": "Disk Usage" - }, - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "displayName", - "value": "Inode Usage" - }, - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "device" - }, - "properties": [ - { - "id": "displayName", - "value": "Device" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "displayName", - "value": "Instance" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 15, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "expr": "sum by (instance, device) (container_fs_usage_bytes{job=~\"$job\", instance=~\"$instance\", id=\"/\", device=~\"/dev/.+\"} / container_fs_limit_bytes{job=~\"$job\", instance=~\"$instance\", id=\"/\", device=~\"/dev/.+\"})", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - }, - { - "datasource": { - "uid": "$prometheus_datasource" - }, - "expr": "1 - sum by (instance, device) (container_fs_inodes_free{job=~\"$job\", instance=~\"$instance\", id=\"/\", device=~\"/dev/.+\"} / container_fs_inodes_total{job=~\"$job\", instance=~\"$instance\", id=\"/\", device=~\"/dev/.+\"})", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B" - } - ], - "transformations": [ - { - "id": "merge", - "options": { } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "device": false - }, - "includeByName": { }, - "indexByName": { }, - "renameByName": { } - } - } - ], - "type": "table" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "docker-integration" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "label": "Data Source", - "multi": false, - "name": "prometheus_datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "(?!grafanacloud-usage|grafanacloud-ml-metrics).+", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "$prometheus_datasource" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "options": [ ], - "query": "label_values(machine_scrape_error, job)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "$prometheus_datasource" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Instance", - "multi": true, - "name": "instance", - "options": [ ], - "query": "label_values(machine_scrape_error{job=~\"$job\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "$prometheus_datasource" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Container", - "multi": true, - "name": "container", - "options": [ ], - "query": "label_values(container_last_seen{job=~\"$job\", instance=~\"$instance\"}, name)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "default", - "title": "Docker Overview", - "uid": "integration-docker-overview", - "version": 1, - "weekStart": "" -} diff --git a/assets/docker/rules.yaml b/assets/docker/rules.yaml index 19765bd..0967ef4 100644 --- a/assets/docker/rules.yaml +++ b/assets/docker/rules.yaml @@ -1 +1 @@ -null +{} diff --git a/assets/etcd/alerts.yaml b/assets/etcd/alerts.yaml index 9c639e3..a7dad08 100644 --- a/assets/etcd/alerts.yaml +++ b/assets/etcd/alerts.yaml @@ -150,7 +150,7 @@ groups: quota as the writes to etcd will be disabled when it is full.' summary: etcd cluster database is running full. expr: | - (last_over_time(etcd_mvcc_db_total_size_in_bytes[5m]) / last_over_time(etcd_server_quota_backend_bytes[5m]))*100 > 95 + (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95 for: 10m labels: severity: critical @@ -162,7 +162,7 @@ groups: be disruptive.' summary: etcd cluster database growing very fast. expr: | - predict_linear(etcd_mvcc_db_total_size_in_bytes[4h], 4*60*60) > etcd_server_quota_backend_bytes + predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"} for: 10m labels: severity: warning @@ -176,7 +176,7 @@ groups: summary: etcd database size in use is less than 50% of the actual allocated storage. expr: | - (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes > 104857600 + (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600 for: 10m labels: severity: warning diff --git a/assets/etcd/dashboards/etcd.json b/assets/etcd/dashboards/etcd.json index d8fea7e..576b096 100644 --- a/assets/etcd/dashboards/etcd.json +++ b/assets/etcd/dashboards/etcd.json @@ -1,1169 +1,531 @@ { - "annotations": { - "list": [ ] - }, "description": "etcd sample Grafana dashboard with Prometheus", - "editable": true, - "gnetId": null, - "hideControls": false, - "links": [ ], - "refresh": "10s", - "rows": [ + "panels": [ { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "$datasource", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 28, - "interval": null, - "isNew": true, - "links": [ ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", - "intervalFactor": 2, - "legendFormat": "", - "metric": "etcd_server_has_leader", - "refId": "A", - "step": 20 - } - ], - "thresholds": "", - "title": "Up", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "aliasColors": { }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 23, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "span": 5, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "RPC Rate", - "metric": "grpc_server_started_total", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "RPC Failed Rate", - "metric": "grpc_server_handled_total", - "refId": "B", - "step": 2 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "RPC Rate", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 41, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "span": 4, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", - "intervalFactor": 2, - "legendFormat": "Watch Streams", - "metric": "grpc_server_handled_total", - "refId": "A", - "step": 4 - }, - { - "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", - "intervalFactor": 2, - "legendFormat": "Lease Streams", - "metric": "grpc_server_handled_total", - "refId": "B", - "step": 4 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Active Streams", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": "1m", + "options": { + "colorMode": "none", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" ] } - ], - "showTitle": false, - "title": "Row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ + }, + "pluginVersion": "v10.0.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "datasource": "$datasource", - "decimals": null, - "editable": true, - "error": false, - "fill": 0, - "grid": { }, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 2, - "links": [ ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} DB Size", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "DB Size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "grid": { }, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 1, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "span": 4, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}} WAL fsync", - "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", - "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", - "intervalFactor": 2, - "legendFormat": "{{instance}} DB fsync", - "metric": "etcd_disk_backend_commit_duration_seconds_bucket", - "refId": "B", - "step": 4 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Disk Sync Duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 29, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_resident_memory_bytes{job=\"$cluster\"}", - "intervalFactor": 2, - "legendFormat": "{{instance}} Resident Memory", - "metric": "process_resident_memory_bytes", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Memory", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "sum(etcd_server_has_leader{job=~\".*etcd.*\", job=\"$cluster\"})", + "legendFormat": "{{cluster}} - {{namespace}}\n" } ], - "title": "New row" + "title": "Up", + "type": "stat" }, { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 7, + "w": 10, + "x": 6, + "y": 0 + }, + "id": 2, + "interval": "1m", + "pluginVersion": "v10.0.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 5, - "id": 22, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 2, - "links": [ ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "span": 3, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[$__rate_interval])", - "intervalFactor": 2, - "legendFormat": "{{instance}} Client Traffic In", - "metric": "etcd_network_client_grpc_received_bytes_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Client Traffic In", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "sum(rate(grpc_server_started_total{job=~\".*etcd.*\", job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))", + "legendFormat": "RPC rate" }, { - "aliasColors": { }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 5, - "id": 21, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 2, - "links": [ ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "span": 3, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])", - "intervalFactor": 2, - "legendFormat": "{{instance}} Client Traffic Out", - "metric": "etcd_network_client_grpc_sent_bytes_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Client Traffic Out", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 20, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", - "intervalFactor": 2, - "legendFormat": "{{instance}} Peer Traffic In", - "metric": "etcd_network_peer_received_bytes_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Peer Traffic In", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "datasource": "$datasource", - "decimals": null, - "editable": true, - "error": false, - "fill": 0, - "grid": { }, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} Peer Traffic Out", - "metric": "etcd_network_peer_sent_bytes_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Peer Traffic Out", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "sum(rate(grpc_server_handled_total{job=~\".*etcd.*\", job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))", + "legendFormat": "RPC failed rate" } ], - "title": "New row" + "title": "RPC rate", + "type": "timeseries" }, { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 40, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[$__rate_interval]))", - "intervalFactor": 2, - "legendFormat": "Proposal Failure Rate", - "metric": "etcd_server_proposals_failed_total", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", - "intervalFactor": 2, - "legendFormat": "Proposal Pending Total", - "metric": "etcd_server_proposals_pending", - "refId": "B", - "step": 2 - }, - { - "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[$__rate_interval]))", - "intervalFactor": 2, - "legendFormat": "Proposal Commit Rate", - "metric": "etcd_server_proposals_committed_total", - "refId": "C", - "step": 2 - }, - { - "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[$__rate_interval]))", - "intervalFactor": 2, - "legendFormat": "Proposal Apply Rate", - "refId": "D", - "step": 2 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Raft Proposals", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "datasource": "$datasource", - "decimals": 0, - "editable": true, - "error": false, - "fill": 0, - "id": 19, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", - "intervalFactor": 2, - "legendFormat": "{{instance}} Total Leader Elections Per Day", - "metric": "etcd_server_leader_changes_seen_total", - "refId": "A", - "step": 2 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Total Leader Elections Per Day", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "decimals": 0, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": { } - }, - "overrides": [ ] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 28 - }, - "hiddenSeries": false, - "id": 42, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ ], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"$cluster\"}[$__rate_interval])))", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} Peer round trip time", - "metric": "etcd_network_peer_round_trip_time_seconds_bucket", - "refId": "A", - "step": 2 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeRegions": [ ], - "timeShift": null, - "title": "Peer round trip time", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "$$hashKey": "object:925", - "decimals": null, - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:926", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "lineWidth": 2, + "showPoints": "never" } } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 3, + "interval": "1m", + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(grpc_server_started_total{job=~\".*etcd.*\",job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "legendFormat": "Watch streams" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(grpc_server_started_total{job=~\".*etcd.*\",job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "legendFormat": "Lease streams" + } ], - "title": "New row" + "title": "Active streams", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "bytes" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 25 + }, + "id": 4, + "interval": "1m", + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "etcd_mvcc_db_total_size_in_bytes{job=~\".*etcd.*\", job=\"$cluster\"}", + "legendFormat": "{{instance}} DB size" + } + ], + "title": "DB size", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "s" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 25 + }, + "id": 5, + "interval": "1m", + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "legendFormat": "{{instance}} WAL fsync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "legendFormat": "{{instance}} DB fsync" + } + ], + "title": "Disk sync duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "bytes" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 25 + }, + "id": 6, + "interval": "1m", + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "process_resident_memory_bytes{job=~\".*etcd.*\", job=\"$cluster\"}", + "legendFormat": "{{instance}} resident memory" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "Bps" + } + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 50 + }, + "id": 7, + "interval": "1m", + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])", + "legendFormat": "{{instance}} client traffic in" + } + ], + "title": "Client traffic in", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "Bps" + } + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 50 + }, + "id": 8, + "interval": "1m", + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])", + "legendFormat": "{{instance}} client traffic out" + } + ], + "title": "Client traffic out", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "Bps" + } + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 50 + }, + "id": 9, + "interval": "1m", + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance)", + "legendFormat": "{{instance}} peer traffic in" + } + ], + "title": "Peer traffic in", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "Bps" + } + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 50 + }, + "id": 10, + "interval": "1m", + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance)", + "legendFormat": "{{instance}} peer traffic out" + } + ], + "title": "Peer traffic out", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "lineWidth": 2, + "showPoints": "never" + } + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 75 + }, + "id": 11, + "interval": "1m", + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "changes(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\", job=\"$cluster\"}[1d])", + "legendFormat": "{{instance}} total leader elections per day" + } + ], + "title": "Raft proposals", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "lineWidth": 2, + "showPoints": "never" + } + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 75 + }, + "id": 12, + "interval": "1m", + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "changes(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\", job=\"$cluster\"}[1d])", + "legendFormat": "{{instance}} total leader elections per day" + } + ], + "title": "Total leader elections per day", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "s" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 75 + }, + "id": 13, + "interval": "1m", + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])))", + "legendFormat": "{{instance}} peer round trip time" + } + ], + "title": "Peer round trip time", + "type": "timeseries" } ], - "schemaVersion": 13, - "sharedCrosshair": false, - "style": "dark", + "refresh": "10s", + "schemaVersion": 36, "tags": [ "etcd-mixin" ], "templating": { "list": [ { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, "label": "Data Source", "name": "datasource", - "options": [ ], "query": "prometheus", - "refresh": 1, - "regex": "", "type": "datasource" }, { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, "label": "cluster", - "multi": false, "name": "cluster", - "options": [ ], - "query": "label_values(etcd_server_has_leader, job)", + "query": "label_values(etcd_server_has_leader{job=~\".*etcd.*\"}, job)", "refresh": 2, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" } ] }, @@ -1171,34 +533,7 @@ "from": "now-15m", "to": "now" }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", + "timezone": "utc", "title": "etcd", - "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", - "version": 215 + "uid": "c2f4e12cdf69feb95caa41a5a1b423d9" } diff --git a/assets/gitlab/dashboards/gitlab-overview.json b/assets/gitlab/dashboards/gitlab-overview.json index 635530f..ba24e49 100644 --- a/assets/gitlab/dashboards/gitlab-overview.json +++ b/assets/gitlab/dashboards/gitlab-overview.json @@ -819,7 +819,7 @@ "useTags": false }, { - "allValue": "", + "allValue": ".+", "current": { }, "datasource": { "uid": "${prometheus_datasource}" diff --git a/assets/gluster/dashboards/k8s-storage-resources-glusterfs-pv.json b/assets/gluster/dashboards/k8s-storage-resources-glusterfs-pv.json deleted file mode 100644 index a2887fa..0000000 --- a/assets/gluster/dashboards/k8s-storage-resources-glusterfs-pv.json +++ /dev/null @@ -1,2288 +0,0 @@ -{ - "annotations": { - "list": [ ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "colorValue": true, - "colors": [ - "#cc0000", - "#92d400", - "#FFFFFF" - ], - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "height": "75px", - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "gluster_volume_up{volume = \"$Volume\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "0.9,1.1", - "timeFrom": null, - "timeShift": null, - "title": "Status", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "valueMaps": [ - { - "op": "=", - "text": "DOWN", - "value": "0" - }, - { - "op": "=", - "text": "UP", - "value": "1" - } - ], - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "height": "75px", - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "gluster:volume_capacity_used_bytes_total:sum{volume='$Volume'} / gluster:volume_capacity_total_bytes:sum{volume='$Volume'}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Capacity Utilization", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "decbytes", - "height": "75px", - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "gluster:volume_capacity_total_bytes:sum{volume='$Volume'} - gluster:volume_capacity_used_bytes_total:sum{volume='$Volume'}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Available Capacity", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "height": "75px", - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(sum (gluster_volume_profile_total_reads_interval{volume='$Volume'}) + sum(gluster_volume_profile_total_writes_interval{volume='$Volume'}))/max(gluster_volume_profile_duration_secs_interval{volume='$Volume'})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "IOPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "gluster:volume_capacity_used_bytes_total:sum{volume='$Volume'}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{$Volume}}", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Capacity Utilization", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "(sum (gluster_volume_profile_total_reads_interval{volume='$Volume'}) + sum(gluster_volume_profile_total_writes_interval{volume='$Volume'}))/max(gluster_volume_profile_duration_secs_interval{volume='$Volume'})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{$Volume}}", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "IOPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Volume Summary", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "100*(gluster_thinpool_metadata_used_bytes{volume='$Volume'} / gluster_thinpool_metadata_total_bytes{volume='$Volume'})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{thinpool_name}}", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "LVM Thin Pool Metadata %", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "100*(gluster_thinpool_data_used_bytes{volume='$Volume'} / gluster_thinpool_data_total_bytes{volume='$Volume'})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{thinpool_name}}", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "LVM Thin Pool Data Usage %", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "LVM Summary", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "height": "75px", - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "gluster_volume_brick_count{volume='$Volume'}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Bricks-Total", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "colorValue": true, - "colors": [ - "#FFFFFF", - "#FFFFFF", - "#92d400" - ], - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "height": "75px", - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "count (gluster_brick_up{volume='$Volume'} == 1)", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "0.1,0.9", - "timeFrom": null, - "timeShift": null, - "title": "Bricks-Up", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "colorValue": true, - "colors": [ - "#FFFFFF", - "#FFFFFF", - "#cc0000" - ], - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "height": "75px", - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "count(gluster_brick_up{volume='$Volume'}) - count (gluster_brick_up{volume='$Volume'} == 1)", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "0.1,0.9", - "timeFrom": null, - "timeShift": null, - "title": "Bricks-Down", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "height": "75px", - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg (gluster_brick_capacity_used_bytes{volume='$Volume'} / gluster_brick_capacity_bytes_total{volume='$Volume'})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Capacity Utilization-Average", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "height": "75px", - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "min (gluster_brick_capacity_used_bytes{volume='$Volume'} / gluster_brick_capacity_bytes_total{volume='$Volume'})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Capacity Utilization-Min", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "height": "75px", - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max (gluster_brick_capacity_used_bytes{volume='$Volume'} / gluster_brick_capacity_bytes_total{volume='$Volume'})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Capacity Utilization-Max", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "height": "75px", - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "count((gluster_brick_capacity_used_bytes{volume='$Volume'} / gluster_brick_capacity_bytes_total{volume='$Volume'}) < 0.8)", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Brick Usage < 80%", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "height": "75px", - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "count((gluster_brick_capacity_used_bytes{volume='$Volume'} / gluster_brick_capacity_bytes_total{volume='$Volume'}) < 0.9) - count((gluster_brick_capacity_used_bytes{volume='$Volume'} / gluster_brick_capacity_bytes_total{volume='$Volume'}) < 0.8)", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Brick Usage > 80% < 90%", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "height": "75px", - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "count(gluster_brick_capacity_used_bytes{volume='$Volume'}) - count((gluster_brick_capacity_used_bytes{volume='$Volume'} / gluster_brick_capacity_bytes_total{volume='$Volume'}) < 0.9)", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Brick Usage > 90%", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "height": "75px", - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg((gluster_volume_profile_total_reads_interval{volume='$Volume'} + gluster_volume_profile_total_writes_interval{volume='$Volume'}) /gluster_volume_profile_duration_secs_interval{volume='$Volume'})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "IOPS - Avg", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "height": "75px", - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "min((gluster_volume_profile_total_reads_interval{volume='$Volume'} + gluster_volume_profile_total_writes_interval{volume='$Volume'}) /gluster_volume_profile_duration_secs_interval{volume='$Volume'})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "IOPS - Min", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "height": "75px", - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max((gluster_volume_profile_total_reads_interval{volume='$Volume'} + gluster_volume_profile_total_writes_interval{volume='$Volume'}) /gluster_volume_profile_duration_secs_interval{volume='$Volume'})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "IOPS - Max", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "height": "75px", - "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum (gluster_volume_heal_count{volume='$Volume'})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Heal Counts-Total Files", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "none", - "height": "75px", - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum (gluster_volume_split_brain_heal_count{volume='$Volume'})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Heal Counts-Files in Split-Brain", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Bricks (Volume Subcomponent) Summary", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Avg Latency", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value", - "thresholds": [ ], - "type": "number", - "unit": "short" - }, - { - "alias": "File Operations", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "fop", - "thresholds": [ ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sort(sum(gluster_volume_profile_fop_avg_latency{volume='$Volume', fop!=''})by (fop))", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - }, - { - "expr": "", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Top file operation", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(gluster_volume_profile_fop_total_hits_on_aggregated_fops{volume='$Volume',fop='LOCK_OPS'}) by (volume)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "File Operations For Locks Trends", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(gluster_volume_profile_fop_total_hits_on_aggregated_fops{volume='$Volume',fop='READ_WRITE_OPS'}) by (volume)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "File Operations for Read/Write", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(gluster_volume_profile_fop_total_hits_on_aggregated_fops{volume='$Volume',fop='INODE_OPS'}) by (volume)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "File Operations for Inode Operations", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 27, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(gluster_volume_profile_fop_total_hits_on_aggregated_fops{volume='$Volume',fop='ENTRY_OPS'}) by (volume)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "File Operations for Entry Operations", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Volume Profiling", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ ], - "templating": { - "list": [ - { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data Source", - "name": "datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Volume", - "multi": false, - "name": "Volume", - "options": [ ], - "query": "label_values(gluster_volume_up, volume)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "K8s / Storage Resources / Persistent Volume / glusterfs", - "uid": "47086bc22842dfaacc6eeec2d230e4ab", - "version": 0 -} diff --git a/assets/grafana/alerts.yaml b/assets/grafana/alerts.yaml index ac37e43..218efa8 100644 --- a/assets/grafana/alerts.yaml +++ b/assets/grafana/alerts.yaml @@ -6,8 +6,8 @@ groups: message: '{{ $labels.namespace }}/{{ $labels.job }}/{{ $labels.handler }} is experiencing {{ $value | humanize }}% errors' expr: | - 100 * namespace_job_handler_statuscode:grafana_http_request_duration_seconds_count:rate5m{handler!~"/api/datasources/proxy/:id.*|/api/ds/query|/api/tsdb/query", status_code=~"5.."} - / ignoring (status_code) + 100 * sum without (status_code) (namespace_job_handler_statuscode:grafana_http_request_duration_seconds_count:rate5m{handler!~"/api/datasources/proxy/:id.*|/api/ds/query|/api/tsdb/query", status_code=~"5.."}) + / sum without (status_code) (namespace_job_handler_statuscode:grafana_http_request_duration_seconds_count:rate5m{handler!~"/api/datasources/proxy/:id.*|/api/ds/query|/api/tsdb/query"}) > 50 for: 5m diff --git a/assets/grafana/dashboards/grafana-overview.json b/assets/grafana/dashboards/grafana-overview.json index 7a9f6d4..6e9ce5d 100644 --- a/assets/grafana/dashboards/grafana-overview.json +++ b/assets/grafana/dashboards/grafana-overview.json @@ -438,7 +438,6 @@ } ], "schemaVersion": 30, - "style": "dark", "tags": [ ], "templating": { "list": [ diff --git a/assets/haproxy/alerts.yaml b/assets/haproxy/alerts.yaml index f8b5066..78a14fd 100644 --- a/assets/haproxy/alerts.yaml +++ b/assets/haproxy/alerts.yaml @@ -6,7 +6,7 @@ groups: description: HAProxy {{$labels.job}} on {{$labels.instance}} is dropping logs. summary: HAProxy is dropping logs. expr: rate(haproxy_process_dropped_logs_total[5m]) != 0 - for: 5s + for: 10m labels: severity: critical - alert: HAProxyBackendCheckFlapping @@ -15,7 +15,7 @@ groups: has flapping checks. summary: HAProxy backend checks are flapping. expr: rate(haproxy_backend_check_up_down_total[5m]) != 0 - for: 1m + for: 10m labels: severity: critical - alert: HAProxyServerCheckFlapping @@ -24,6 +24,6 @@ groups: has flapping checks. summary: HAProxy server checks are flapping. expr: rate(haproxy_server_check_up_down_total[5m]) != 0 - for: 1m + for: 10m labels: severity: critical diff --git a/assets/ibm-mq/dashboards/ibm-mq-queue-manager-overview.json b/assets/ibm-mq/dashboards/ibm-mq-queue-manager-overview.json index 394eb1b..42bfe3a 100644 --- a/assets/ibm-mq/dashboards/ibm-mq-queue-manager-overview.json +++ b/assets/ibm-mq/dashboards/ibm-mq-queue-manager-overview.json @@ -1445,7 +1445,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{job=~\"$job\", filename=~\"/var/mqm/qmgrs/.*/errors/.*LOG\", qmgr=~\"$qmgr\"} |= ``", + "expr": "{job=~\"$job\", qmgr=~\"$qmgr\"} |= `` | (filename=~\"/var/mqm/qmgrs/.*/errors/.*LOG\" or log_type=\"mq-qmgr-error\")", "queryType": "range", "refId": "A" } diff --git a/assets/influxdb/dashboards/influxdb-logs.json b/assets/influxdb/dashboards/influxdb-logs.json index edb2d18..ef7d43b 100644 --- a/assets/influxdb/dashboards/influxdb-logs.json +++ b/assets/influxdb/dashboards/influxdb-logs.json @@ -142,17 +142,19 @@ }, "gridPos": { "h": 6, - "w": 24 + "w": 24, + "x": 0, + "y": 0 }, "id": 1, - "interval": "30s", + "maxDataPoints": 100, "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -182,7 +184,9 @@ }, "gridPos": { "h": 18, - "w": 24 + "w": 24, + "x": 0, + "y": 18 }, "id": 2, "options": { @@ -192,7 +196,7 @@ "showTime": false, "wrapLogMessage": true }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -207,7 +211,7 @@ } ], "refresh": "1m", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "influxdb-mixin" ], diff --git a/assets/istio/dashboards/logs b/assets/istio/dashboards/logs index 59375b5..131fcc8 100644 --- a/assets/istio/dashboards/logs +++ b/assets/istio/dashboards/logs @@ -147,17 +147,19 @@ }, "gridPos": { "h": 6, - "w": 24 + "w": 24, + "x": 0, + "y": 0 }, "id": 1, - "interval": "30s", + "maxDataPoints": 100, "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -187,7 +189,9 @@ }, "gridPos": { "h": 18, - "w": 24 + "w": 24, + "x": 0, + "y": 18 }, "id": 2, "options": { @@ -197,7 +201,7 @@ "showTime": false, "wrapLogMessage": false }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -212,7 +216,7 @@ } ], "refresh": "1m", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "istio" ], diff --git a/assets/istio/dashboards/overview b/assets/istio/dashboards/overview index 11e9160..8e53201 100644 --- a/assets/istio/dashboards/overview +++ b/assets/istio/dashboards/overview @@ -56,7 +56,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -105,7 +105,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -154,7 +154,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -222,7 +222,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -271,7 +271,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -320,7 +320,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -372,7 +372,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -442,7 +442,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -543,7 +543,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -693,7 +693,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -787,7 +787,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -1123,7 +1123,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -1183,7 +1183,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { diff --git a/assets/istio/dashboards/servicesOverview b/assets/istio/dashboards/servicesOverview index f0d1755..500dfe3 100644 --- a/assets/istio/dashboards/servicesOverview +++ b/assets/istio/dashboards/servicesOverview @@ -72,7 +72,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -122,7 +122,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -177,7 +177,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -231,7 +231,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -338,7 +338,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -482,7 +482,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -537,7 +537,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -591,7 +591,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -657,7 +657,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -707,7 +707,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -762,7 +762,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -816,7 +816,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -923,7 +923,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -1067,7 +1067,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -1122,7 +1122,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -1176,7 +1176,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { diff --git a/assets/istio/dashboards/workloadsOverview b/assets/istio/dashboards/workloadsOverview index 3690380..1b9d4a0 100644 --- a/assets/istio/dashboards/workloadsOverview +++ b/assets/istio/dashboards/workloadsOverview @@ -72,7 +72,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -122,7 +122,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -177,7 +177,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -231,7 +231,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -338,7 +338,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -482,7 +482,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -537,7 +537,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -591,7 +591,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -657,7 +657,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -707,7 +707,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -762,7 +762,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -816,7 +816,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -923,7 +923,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -1067,7 +1067,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -1122,7 +1122,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -1176,7 +1176,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { diff --git a/assets/jaeger/alerts.yaml b/assets/jaeger/alerts.yaml index 40bbde5..3340763 100644 --- a/assets/jaeger/alerts.yaml +++ b/assets/jaeger/alerts.yaml @@ -3,16 +3,18 @@ groups: rules: - alert: JaegerAgentUDPPacketsBeingDropped annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }} UDP packets per second. + summary: Jaeger agent is dropping UDP packets. expr: rate(jaeger_agent_thrift_udp_server_packets_dropped_total[1m]) > 1 for: 15m labels: severity: warning - alert: JaegerAgentHTTPServerErrs annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is experiencing {{ printf "%.2f" $value }}% HTTP errors. + summary: Jaeger agent is experiencing HTTP errors. expr: 100 * sum(rate(jaeger_agent_http_server_errors_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_agent_http_server_total[1m])) by (instance, job, namespace)> 1 @@ -21,8 +23,9 @@ groups: severity: warning - alert: JaegerClientSpansDropped annotations: - message: | + description: | service {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }}% spans. + summary: Jaeger client is dropping spans. expr: 100 * sum(rate(jaeger_reporter_spans{result=~"dropped|err"}[1m])) by (instance, job, namespace) / sum(rate(jaeger_reporter_spans[1m])) by (instance, job, namespace)> 1 @@ -31,8 +34,9 @@ groups: severity: warning - alert: JaegerAgentSpansDropped annotations: - message: | + description: | agent {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }}% spans. + summary: Jaeger agent is dropping spans. expr: 100 * sum(rate(jaeger_agent_reporter_batches_failures_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_agent_reporter_batches_submitted_total[1m])) by (instance, job, namespace)> 1 @@ -41,16 +45,18 @@ groups: severity: warning - alert: JaegerCollectorQueueNotDraining annotations: - message: | + description: | collector {{ $labels.job }} {{ $labels.instance }} is not able to drain the queue. + summary: Jaeger collector is not able to drain the queue. expr: avg_over_time(jaeger_collector_queue_length[10m]) > 1000 for: 15m labels: severity: warning - alert: JaegerCollectorDroppingSpans annotations: - message: | + description: | collector {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }}% spans. + summary: Jaeger collector is dropping spans. expr: 100 * sum(rate(jaeger_collector_spans_dropped_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_collector_spans_received_total[1m])) by (instance, job, namespace)> 1 @@ -59,8 +65,9 @@ groups: severity: warning - alert: JaegerSamplingUpdateFailing annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is failing {{ printf "%.2f" $value }}% in updating sampling policies. + summary: Jaeger's sampling update is failing. expr: 100 * sum(rate(jaeger_sampler_queries{result="err"}[1m])) by (instance, job, namespace) / sum(rate(jaeger_sampler_queries[1m])) by (instance, job, namespace)> 1 @@ -69,8 +76,9 @@ groups: severity: warning - alert: JaegerCollectorPersistenceSlow annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is slow at persisting spans. + summary: Jaeger collector is slow at persisting spans. expr: histogram_quantile(0.99, sum by (le) (rate(jaeger_collector_save_latency_bucket[1m]))) > 0.5 for: 15m @@ -78,8 +86,9 @@ groups: severity: warning - alert: JaegerThrottlingUpdateFailing annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is failing {{ printf "%.2f" $value }}% in updating throttling policies. + summary: Jaeger's throttling update is failing. expr: 100 * sum(rate(jaeger_throttler_updates{result="err"}[1m])) by (instance, job, namespace) / sum(rate(jaeger_throttler_updates[1m])) by (instance, job, namespace)> 1 @@ -88,8 +97,9 @@ groups: severity: warning - alert: JaegerQueryReqsFailing annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is seeing {{ printf "%.2f" $value }}% query errors on {{ $labels.operation }}. + summary: Jaeger queries are failing. expr: 100 * sum(rate(jaeger_query_requests_total{result="err"}[1m])) by (instance, job, namespace) / sum(rate(jaeger_query_requests_total[1m])) by (instance, job, namespace)> 1 @@ -98,8 +108,9 @@ groups: severity: warning - alert: JaegerCassandraWritesFailing annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is seeing {{ printf "%.2f" $value }}% query errors on {{ $labels.operation }}. + summary: Jaeger writes to Cassandra are failing. expr: 100 * sum(rate(jaeger_cassandra_errors_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_cassandra_attempts_total[1m])) by (instance, job, namespace)> 1 @@ -108,8 +119,9 @@ groups: severity: warning - alert: JaegerCassandraReadsFailing annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is seeing {{ printf "%.2f" $value }}% query errors on {{ $labels.operation }}. + summary: Jaeger reads from Cassandra are failing. expr: 100 * sum(rate(jaeger_cassandra_read_errors_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_cassandra_read_attempts_total[1m])) by (instance, job, namespace)> 1 diff --git a/assets/jaeger/dashboards/jaeger-read.json b/assets/jaeger/dashboards/jaeger-read.json index 2b1d3ed..b115d99 100644 --- a/assets/jaeger/dashboards/jaeger-read.json +++ b/assets/jaeger/dashboards/jaeger-read.json @@ -50,7 +50,6 @@ { "expr": "sum(rate(jaeger_query_requests_total{result=\"err\"}[1m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "error", "refId": "A", "step": 10 @@ -58,7 +57,6 @@ { "expr": "sum(rate(jaeger_query_requests_total[1m])) - sum(rate(jaeger_query_requests_total{result=\"err\"}[1m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "success", "refId": "B", "step": 10 @@ -134,7 +132,6 @@ { "expr": "histogram_quantile(0.99, sum(rate(jaeger_query_latency_bucket[1m])) by (le, instance))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{instance}}", "legendLink": null } @@ -224,7 +221,6 @@ { "expr": "sum(rate(jaeger_cassandra_read_errors_total[1m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "error", "refId": "A", "step": 10 @@ -232,7 +228,6 @@ { "expr": "sum(rate(jaeger_cassandra_read_attempts_total[1m])) - sum(rate(jaeger_cassandra_read_errors_total[1m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "success", "refId": "B", "step": 10 @@ -308,7 +303,6 @@ { "expr": "histogram_quantile(0.99, sum(rate(jaeger_cassandra_read_latency_ok_bucket[1m])) by (le, instance))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{instance}}", "legendLink": null } @@ -369,7 +363,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/jaeger/dashboards/jaeger-write.json b/assets/jaeger/dashboards/jaeger-write.json index 21dafe7..f3320f8 100644 --- a/assets/jaeger/dashboards/jaeger-write.json +++ b/assets/jaeger/dashboards/jaeger-write.json @@ -50,7 +50,6 @@ { "expr": "sum(rate(jaeger_reporter_spans{result=~\"dropped|err\"}[1m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "error", "refId": "A", "step": 10 @@ -58,7 +57,6 @@ { "expr": "sum(rate(jaeger_reporter_spans[1m])) - sum(rate(jaeger_reporter_spans{result=~\"dropped|err\"}[1m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "success", "refId": "B", "step": 10 @@ -134,7 +132,6 @@ { "expr": "sum(rate(jaeger_reporter_spans{result=~\"dropped|err\"}[1m])) by (namespace) / scalar(sum(rate(jaeger_reporter_spans[1m])))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{namespace}}", "legendLink": null } @@ -224,7 +221,6 @@ { "expr": "sum(rate(jaeger_agent_reporter_batches_failures_total[1m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "error", "refId": "A", "step": 10 @@ -232,7 +228,6 @@ { "expr": "sum(rate(jaeger_agent_reporter_batches_submitted_total[1m])) - sum(rate(jaeger_agent_reporter_batches_failures_total[1m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "success", "refId": "B", "step": 10 @@ -308,7 +303,6 @@ { "expr": "sum(rate(jaeger_agent_reporter_batches_failures_total[1m])) by (cluster) / scalar(sum(rate(jaeger_agent_reporter_batches_submitted_total[1m])))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{cluster}}", "legendLink": null } @@ -398,7 +392,6 @@ { "expr": "sum(rate(jaeger_collector_spans_dropped_total[1m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "error", "refId": "A", "step": 10 @@ -406,7 +399,6 @@ { "expr": "sum(rate(jaeger_collector_spans_received_total[1m])) - sum(rate(jaeger_collector_spans_dropped_total[1m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "success", "refId": "B", "step": 10 @@ -482,7 +474,6 @@ { "expr": "sum(rate(jaeger_collector_spans_dropped_total[1m])) by (instance) / scalar(sum(rate(jaeger_collector_spans_received_total[1m])))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{instance}}", "legendLink": null } @@ -569,7 +560,6 @@ { "expr": "sum by (svc) (rate(jaeger_collector_traces_saved_by_svc_total[1m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{svc}}", "legendLink": null } @@ -644,7 +634,6 @@ { "expr": "sum by (svc) (rate(jaeger_collector_spans_saved_by_svc_total[1m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{svc}}", "legendLink": null } @@ -731,7 +720,6 @@ { "expr": "jaeger_collector_queue_length", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{instance}}", "legendLink": null } @@ -806,7 +794,6 @@ { "expr": "histogram_quantile(0.95, sum(rate(jaeger_collector_in_queue_latency_bucket[1m])) by (le, instance))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{instance}}", "legendLink": null } @@ -896,7 +883,6 @@ { "expr": "sum(rate(jaeger_cassandra_errors_total[1m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "error", "refId": "A", "step": 10 @@ -904,7 +890,6 @@ { "expr": "sum(rate(jaeger_cassandra_attempts_total[1m])) - sum(rate(jaeger_cassandra_errors_total[1m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "success", "refId": "B", "step": 10 @@ -980,7 +965,6 @@ { "expr": "histogram_quantile(0.95, sum by (job, le) (rate(jaeger_collector_save_latency_bucket[1m])))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{job}}", "legendLink": null } @@ -1041,7 +1025,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/jenkins/dashboards/jenkins.json b/assets/jenkins/dashboards/jenkins.json index 50b5100..b07f967 100644 --- a/assets/jenkins/dashboards/jenkins.json +++ b/assets/jenkins/dashboards/jenkins.json @@ -1,1026 +1,1480 @@ { - "__inputs": [ ], - "__requires": [ ], "annotations": { - "list": [ ] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] }, - "editable": false, - "gnetId": null, + "editable": true, + "fiscalYearStartMonth": 0, "graphTooltip": 0, - "hideControls": false, - "id": null, + "id": 607, "links": [ ], - "refresh": "", - "rows": [ + "panels": [ { - "collapse": false, "collapsed": false, - "panels": [ + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 14, + "panels": [ ], + "targets": [ { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "description": "Total number of job executors", - "format": "short", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "gridPos": { }, - "id": 2, - "interval": null, - "links": [ ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "jenkins_executor_count_value{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Total Executors", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "aliasColors": { }, - "datasource": "$datasource", - "description": "Shows distribution of executors by their current state", - "gridPos": { }, - "id": 3, - "legend": { - "percentage": false, - "show": true, - "values": true - }, - "legendType": "Under graph", - "pieType": "pie", - "span": 2, - "targets": [ - { - "expr": "jenkins_executor_free_value{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "legendFormat": "free", - "refId": "A" - }, - { - "expr": "jenkins_executor_in_use_value{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "legendFormat": "in-use", - "refId": "B" - } - ], - "title": "Executors By State", - "type": "piechart", - "valueName": "current" - }, - { - "datasource": "$datasource", - "description": "Rate of jobs that are successful over the specified interval.", - "fieldConfig": { - "defaults": { - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": 0 - } - ] - }, - "unit": "none" - } - }, - "gridPos": { }, - "id": 4, - "links": [ ], - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7", - "span": 2, - "targets": [ - { - "expr": "rate(jenkins_runs_success_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Job Success Rate", - "transparent": false, - "type": "stat" - }, - { - "datasource": "$datasource", - "description": "Rate of jobs failing over the specified interval.", - "fieldConfig": { - "defaults": { - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": 0 - } - ] - }, - "unit": "none" - } - }, - "gridPos": { }, - "id": 5, - "links": [ ], - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7", - "span": 2, - "targets": [ - { - "expr": "rate(jenkins_runs_failure_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Job Falure Rate", - "transparent": false, - "type": "stat" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "red", - "yellow", - "green" - ], - "datasource": "$datasource", - "description": "Percentage of build nodes online", - "format": "percentunit", - "gauge": { - "maxValue": 1, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { }, - "id": 6, - "interval": null, - "links": [ ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "jenkins_node_online_value{job=~\"$job\", instance=~\"$instance\"}/jenkins_node_count_value{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": ".80,.90", - "title": "Build Nodes Online", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "datasource": "$datasource", - "description": "Plugins grouped by different states.", - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": 0 - } - ] - }, - "unit": null - } - }, - "gridPos": { }, - "id": 7, - "options": { - "orientation": "horizontal" - }, - "span": 2, - "targets": [ - { - "expr": "jenkins_plugins_active{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "legendFormat": "active", - "refId": "A" - }, - { - "expr": "jenkins_plugins_inactive{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "legendFormat": "inactive", - "refId": "B" - }, - { - "expr": "jenkins_plugins_failed{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "legendFormat": "failed", - "refId": "C" - }, - { - "expr": "jenkins_plugins_withUpdate{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "legendFormat": "with update", - "refId": "D" - } - ], - "title": "Plugins by State", - "type": "bargauge" + "refId": "A" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, "title": "Overview", - "titleSize": "h6", "type": "row" }, { - "collapse": false, - "collapsed": false, - "panels": [ - { - "datasource": "$datasource", - "description": "Bottom 5 least healthiest jobs.", - "gridPos": { }, - "id": 8, - "span": 4, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Health", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ ], - "type": "number", - "unit": "percent" - }, - { - "alias": "Failed", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ ], - "type": "number", - "unit": "short" - }, - { - "alias": "Successful", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ ], - "type": "number", - "unit": "short" - }, - { - "alias": "Total", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ ], - "type": "number", - "unit": "short" - }, - { - "alias": "Job", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "jenkins_job", - "thresholds": [ ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "bottomk(5, avg by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_health_score\", job=~\"$job\", instance=~\"$instance\"}))", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "A" - }, - { - "expr": "sum by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_failed_build_count\", job=~\"$job\", instance=~\"$instance\"}) and on (jenkins_job) bottomk(5, avg by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_health_score\", job=~\"$job\", instance=~\"$instance\"}))", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "B" - }, - { - "expr": "sum by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_success_build_count\", job=~\"$job\", instance=~\"$instance\"}) and on (jenkins_job) bottomk(5, avg by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_health_score\", job=~\"$job\", instance=~\"$instance\"}))", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "C" - }, - { - "expr": "sum by (jenkins_job) ({__name__=~\"$prefix\\\\_jenkins_builds_failed_build_count|$prefix\\\\_?jenkins_builds_success_build_count\", job=~\"$job\", instance=~\"$instance\"}) and on (jenkins_job) bottomk(5, avg by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_health_score\", job=~\"$job\", instance=~\"$instance\"}))", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "D" - } - ], - "title": "Least Healthy Jobs", - "transform": "table", - "type": "table" - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Latest duration of the jobs", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false + "datasource": { + "uid": "$datasource" + }, + "description": "Total number of job executors", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ + "mappings": [ { - "expr": "max by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_last_build_duration_milliseconds\", job=~\"$job\", instance=~\"$instance\"})", - "format": "time_series", - "legendFormat": "{{jenkins_job}}", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Latest Build Duration by Job", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "datasource": "$datasource", - "description": "List of the top 5 slowest jobs.", - "gridPos": { }, - "id": 10, - "span": 2, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Duration", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value", - "thresholds": [ ], - "type": "number", - "unit": "ms" - }, - { - "alias": "Job Name", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "jenkins_job", - "thresholds": [ ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "topk(5, max by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_last_build_duration_milliseconds\", job=~\"$job\", instance=~\"$instance\"}))", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Top 5 Slowest Jobs", - "transform": "table", - "type": "table" - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Queued jobs grouped by state.", - "fill": 5, - "fillGradient": 10, - "gridPos": { }, - "id": 11, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "line": 1, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 2, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "jenkins_queue_stuck_value{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "legendFormat": "Stuck", - "legendLink": null - }, - { - "expr": "jenkins_queue_blocked_value{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "legendFormat": "Blocked", - "legendLink": null - }, - { - "expr": "jenkins_queue_pending_value{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "legendFormat": "Pending", - "legendLink": null - }, - { - "expr": "jenkins_queue_buildable_value{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "legendFormat": "Buildable", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Job Queue by State", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Jobs", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Total web requests group by different HTTP status codes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 12, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(http_responseCodes_ok_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "legendFormat": "200 {{instance}}", - "legendLink": null - }, - { - "expr": "rate(http_responseCodes_created_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "legendFormat": "201 {{instance}}", - "legendLink": null - }, - { - "expr": "rate(http_responseCodes_noContent_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "legendFormat": "204 {{instance}}", - "legendLink": null - }, - { - "expr": "rate(http_responseCodes_notModified_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "legendFormat": "304 {{instance}}", - "legendLink": null - }, - { - "expr": "rate(http_responseCodes_badRequest_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "legendFormat": "400 {{instance}}", - "legendLink": null - }, - { - "expr": "rate(http_responseCodes_forbidden_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "legendFormat": "403 {{instance}}", - "legendLink": null - }, - { - "expr": "rate(http_responseCodes_notFound_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "legendFormat": "404 {{instance}}", - "legendLink": null - }, - { - "expr": "rate(http_responseCodes_serverError_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "legendFormat": "500 {{instance}}", - "legendLink": null - }, - { - "expr": "rate(http_responseCodes_serviceUnavailable_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "legendFormat": "503 {{instance}}", - "legendLink": null - }, - { - "expr": "rate(http_responseCodes_other_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "legendFormat": "other {{instance}}", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Web Request Rate by Status", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "datasource": "$datasource", - "description": "Shows a distribution of HTTP request durations by quantile.", - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] + "options": { + "match": "null", + "result": { + "text": "N/A" + } }, - "unit": "s" - } - }, - "gridPos": { }, - "id": 13, - "options": { - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - } - }, - "span": 6, - "targets": [ - { - "expr": "http_requests{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{quantile}}", - "refId": "A" + "type": "special" } ], - "title": "HTTP Request Duration by Quantile", - "type": "bargauge" + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 2, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "jenkins_executor_count_value{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Web UI", - "titleSize": "h6", + "title": "Total executors", + "type": "stat" + }, + { + "aliasColors": { }, + "datasource": { + "uid": "$datasource" + }, + "description": "Shows distribution of executors by their current state", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [ ] + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 3, + "legend": { + "percentage": false, + "show": true, + "values": true + }, + "legendType": "Under graph", + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pieType": "pie", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "jenkins_executor_free_value{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "free", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "jenkins_executor_in_use_value{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "in-use", + "refId": "B" + } + ], + "title": "Executors by state", + "type": "piechart", + "valueName": "current" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Rate of jobs that are successful over the specified interval.", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(jenkins_runs_success_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Job success rate", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Rate of jobs failing over the specified interval.", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(jenkins_runs_failure_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Job failure rate", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Percentage of build nodes online", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.80000000000000004 + }, + { + "color": "green", + "value": 0.90000000000000002 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 6, + "maxDataPoints": 100, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "jenkins_node_online_value{job=~\"$job\", instance=~\"$instance\"}/jenkins_node_count_value{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Build nodes online", + "type": "gauge" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Plugins grouped by different states.", + "fieldConfig": { + "defaults": { + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 7, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "jenkins_plugins_active{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "active", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "jenkins_plugins_inactive{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "inactive", + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "jenkins_plugins_failed{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "failed", + "refId": "C" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "jenkins_plugins_withUpdate{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "with update", + "refId": "D" + } + ], + "title": "Plugins by state", + "type": "bargauge" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 15, + "panels": [ ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "refId": "A" + } + ], + "title": "Jobs", "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Bottom 5 least healthiest jobs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "Health" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "Failed" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "displayName", + "value": "Successful" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #D" + }, + "properties": [ + { + "id": "displayName", + "value": "Total" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "jenkins_job" + }, + "properties": [ + { + "id": "displayName", + "value": "Job" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 9 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "bottomk(5, avg by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_health_score\", job=~\"$job\", instance=~\"$instance\"}))", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_failed_build_count\", job=~\"$job\", instance=~\"$instance\"}) and on (jenkins_job) bottomk(5, avg by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_health_score\", job=~\"$job\", instance=~\"$instance\"}))", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_success_build_count\", job=~\"$job\", instance=~\"$instance\"}) and on (jenkins_job) bottomk(5, avg by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_health_score\", job=~\"$job\", instance=~\"$instance\"}))", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "C" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum by (jenkins_job) ({__name__=~\"$prefix\\\\_jenkins_builds_failed_build_count|$prefix\\\\_?jenkins_builds_success_build_count\", job=~\"$job\", instance=~\"$instance\"}) and on (jenkins_job) bottomk(5, avg by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_health_score\", job=~\"$job\", instance=~\"$instance\"}))", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "D" + } + ], + "title": "Least healthy jobs", + "transformations": [ + { + "id": "seriesToRows", + "options": { + "reducers": [ ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Latest duration of the jobs", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 9 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "max by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_last_build_duration_milliseconds\", job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "legendFormat": "{{jenkins_job}}", + "refId": "A" + } + ], + "title": "Latest build duration by job", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "List of the top 5 slowest jobs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Duration" + }, + { + "id": "unit", + "value": "ms" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "jenkins_job" + }, + "properties": [ + { + "id": "displayName", + "value": "Job Name" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 9 + }, + "id": 10, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "topk(5, max by (jenkins_job) ({__name__=~\"$prefix\\\\_?jenkins_builds_last_build_duration_milliseconds\", job=~\"$job\", instance=~\"$instance\"}))", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top 5 slowest jobs", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [ ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Queued jobs grouped by state.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 9 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "jenkins_queue_stuck_value{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "legendFormat": "Stuck", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "jenkins_queue_blocked_value{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "legendFormat": "Blocked", + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "jenkins_queue_pending_value{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "legendFormat": "Pending", + "refId": "C" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "jenkins_queue_buildable_value{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "legendFormat": "Buildable", + "refId": "D" + } + ], + "title": "Job queue by state", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 16, + "panels": [ ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "refId": "A" + } + ], + "title": "Web UI", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Total web requests group by different HTTP status codes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(http_responseCodes_ok_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "200 {{instance}}", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(http_responseCodes_created_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "201 {{instance}}", + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(http_responseCodes_noContent_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "204 {{instance}}", + "refId": "C" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(http_responseCodes_notModified_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "304 {{instance}}", + "refId": "D" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(http_responseCodes_badRequest_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "400 {{instance}}", + "refId": "E" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(http_responseCodes_forbidden_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "403 {{instance}}", + "refId": "F" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(http_responseCodes_notFound_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "404 {{instance}}", + "refId": "G" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(http_responseCodes_serverError_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "500 {{instance}}", + "refId": "H" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(http_responseCodes_serviceUnavailable_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "503 {{instance}}", + "refId": "I" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(http_responseCodes_other_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "other {{instance}}", + "refId": "J" + } + ], + "title": "Web request rate by status", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Shows a distribution of HTTP request durations by quantile.", + "fieldConfig": { + "defaults": { + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 13, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "http_requests{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{quantile}}", + "refId": "A" + } + ], + "title": "HTTP request duration by quantile", + "type": "bargauge" } ], - "schemaVersion": 14, - "style": "dark", + "refresh": "", + "schemaVersion": 39, "tags": [ ], "templating": { "list": [ { "current": { + "selected": true, "text": "default", "value": "default" }, "hide": 0, - "label": "Data Source", + "includeAll": false, + "label": "Data source", + "multi": false, "name": "datasource", "options": [ ], "query": "prometheus", + "queryValue": "", "refresh": 1, "regex": "", + "skipUrlSync": false, "type": "datasource" }, { "allValue": ".+", - "current": { }, - "datasource": "$datasource", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "", "hide": 0, "includeAll": true, - "label": "job", + "label": "Job", "multi": true, "name": "job", "options": [ ], "query": "label_values(up, job)", "refresh": 1, "regex": "", + "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", - "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".+", - "current": { }, - "datasource": "$datasource", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "", "hide": 0, "includeAll": true, - "label": "instance", + "label": "Instance", "multi": true, "name": "instance", "options": [ ], "query": "label_values(up{job=~\"$job\"}, instance)", "refresh": 1, "regex": "", + "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", - "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false @@ -1031,10 +1485,8 @@ "text": "", "value": "default" }, - "description": null, - "error": null, "hide": 0, - "label": null, + "label": "Prefix", "name": "prefix", "options": [ { @@ -1050,7 +1502,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { @@ -1081,5 +1533,6 @@ "timezone": "browser", "title": "Jenkins", "uid": "2UJijx65J", - "version": 0 + "version": 4, + "weekStart": "" } diff --git a/assets/jvm/alerts.yaml b/assets/jvm/alerts.yaml index a97d41b..19765bd 100644 --- a/assets/jvm/alerts.yaml +++ b/assets/jvm/alerts.yaml @@ -1,14 +1 @@ -groups: -- name: jvm - rules: - - alert: JvmMemoryFillingUp - annotations: - description: JVM memory usage is at {{ printf "%%.0f" $value }} percent over - the last 5 minutes on {{$labels.instance}}, which is above the threshold of - 80%. - summary: JVM memory filling up. - expr: | - jvm_memory_bytes_used / jvm_memory_bytes_max{area="heap"} > 0.8 - for: 5m - labels: - severity: warning +null diff --git a/assets/jvm/dashboards/jvm-dashboard.json b/assets/jvm/dashboards/jvm-dashboard.json deleted file mode 100644 index 6bbd26e..0000000 --- a/assets/jvm/dashboards/jvm-dashboard.json +++ /dev/null @@ -1,577 +0,0 @@ -{ - "__inputs": [ - { - "description": "A prometheus datasource with JMX Exporter scraping", - "label": "Data Source", - "name": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" - } - ], - "__requires": [ - { - "id": "grafana", - "name": "Grafana", - "type": "grafana", - "version": "4.5.0-pre1" - }, - { - "id": "graph", - "name": "Graph", - "type": "panel", - "version": "" - }, - { - "id": "prometheus", - "name": "Prometheus", - "type": "datasource", - "version": "1.0.0" - } - ], - "annotations": { - "list": [ ] - }, - "description": "Dashboard for JVM metrics with Prometheus / JMX Exporter", - "editable": true, - "gnetId": 3066, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [ ], - "refresh": "30s", - "rows": [ - { - "collapse": false, - "height": 250, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${datasource}", - "fill": 1, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "jvm_memory_bytes_used{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "interval": "", - "legendFormat": "{{area}} memory [{{instance}}]", - "metric": "jvm_memory_bytes_used", - "refId": "A", - "step": 5 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Memory used", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 250, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${datasource}", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "jvm_threads_current{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "legendFormat": "current [{{instance}}]", - "metric": "jvm_threads_current", - "refId": "A", - "step": 10 - }, - { - "expr": "jvm_threads_daemon{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "legendFormat": "daemon [{{instance}}]", - "metric": "jvm_threads_daemon", - "refId": "B", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Threads used", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${datasource}", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "jvm_classes_loaded{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "interval": "", - "legendFormat": "loaded [{{instance}}]", - "metric": "jvm_classes_loaded", - "refId": "A", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Class loading", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 250, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${datasource}", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(jvm_gc_collection_seconds_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "legendFormat": "{{gc}} [{{instance}}]", - "metric": "jvm_gc_collection_seconds_sum", - "refId": "A", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "GC time rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${datasource}", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "jvm_gc_collection_seconds_count{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "interval": "", - "legendFormat": "{{gc}} [{{instance}}]", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "GC count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "JVM", - "prometheus" - ], - "templating": { - "list": [ - { - "current": { }, - "hide": 0, - "includeAll": false, - "label": "Data Source", - "multi": false, - "name": "datasource", - "options": [ ], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".+", - "current": { }, - "datasource": "${datasource}", - "hide": 0, - "includeAll": true, - "label": "job", - "multi": true, - "name": "job", - "options": [ ], - "query": "label_values(jvm_memory_bytes_used,job)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { }, - "datasource": "${datasource}", - "hide": 0, - "includeAll": true, - "label": "instance", - "multi": true, - "name": "instance", - "options": [ ], - "query": "label_values(jvm_memory_bytes_used{job=~\"$job\"},instance)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "JVM overview - Prometheus", - "uid": "1602068683-1", - "version": 12 -} diff --git a/assets/jvm/rules.yaml b/assets/jvm/rules.yaml index 19765bd..0967ef4 100644 --- a/assets/jvm/rules.yaml +++ b/assets/jvm/rules.yaml @@ -1 +1 @@ -null +{} diff --git a/assets/kafka/alerts.yaml b/assets/kafka/alerts.yaml index a30194a..19765bd 100644 --- a/assets/kafka/alerts.yaml +++ b/assets/kafka/alerts.yaml @@ -1,101 +1 @@ -groups: -- name: Kafka_Alerts - rules: - - alert: KafkaOfflinePartitonCount - annotations: - description: Kafka cluster {{ $labels.kafka_cluster }} has {{ $value }} offline - partitions. After successful leader election, if the leader for partition - dies, then the partition moves to the OfflinePartition state. Offline partitions - are not available for reading and writing. Restart the brokers, if needed, - and check the logs for errors. - summary: Kafka has offline partitons. - expr: sum without(instance) (kafka_controller_kafkacontroller_offlinepartitionscount{job="integrations/kafka"}) - > 0 - for: 5m - labels: - severity: critical - - alert: KafkaUnderReplicatedPartitionCount - annotations: - description: Kafka instance {{ $labels.instance }} in cluster {{ $labels.kafka_cluster - }} has {{ $value }} under replicated partitons - summary: Kafka has under replicated partitons. - expr: | - sum without() (kafka_server_replicamanager_underreplicatedpartitions{job="integrations/kafka"}) > 0 - for: 5m - labels: - severity: critical - - alert: KafkaActiveController - annotations: - description: Kafka cluster {{ $labels.kafka_cluster }} has {{ $value }} broker(s) - reporting as the active controller in the last 5 minute interval. During steady - state there should be only one active controller per cluster. - summary: Kafka has no active controller. - expr: sum without(instance) (kafka_controller_kafkacontroller_activecontrollercount{job="integrations/kafka"}) - != 1 - for: 5m - labels: - severity: critical - - alert: KafkaUncleanLeaderElection - annotations: - description: Kafka cluster {{ $labels.kafka_cluster }} has {{ $value }} unclean - partition leader elections reported in the last 5 minute interval. When unclean - leader election is held among out-of-sync replicas, there is a possibility - of data loss if any messages were not synced prior to the loss of the former - leader. So if the number of unclean elections is greater than 0, investigate - broker logs to determine why leaders were re-elected, and look for WARN or - ERROR messages. Consider setting the broker configuration parameter unclean.leader.election.enable - to false so that a replica outside of the set of in-sync replicas is never - elected leader. - summary: Kafka has unclean leader elections. - expr: max without(instance) (rate(kafka_controller_controllerstats_uncleanleaderelectionspersec{job="integrations/kafka"}[5m])) - != 0 - for: 5m - labels: - severity: critical - - alert: KafkaISRExpandRate - annotations: - description: Kafka instance {{ $labels.instance }} in cluster {{ $labels.kafka_cluster - }} ISR is expanding by {{ $value }} per second. If a broker goes down, ISR - for some of the partitions shrink. When that broker is up again, ISRs are - expanded once the replicas are fully caught up. Other than that, the expected - value for ISR expansion rate is 0. If ISR is expanding and shrinking frequently, - adjust Allowed replica lag. - summary: Kafka ISR Expansion Rate is expanding. - expr: | - sum without() (rate(kafka_server_replicamanager_isrexpandspersec{job="integrations/kafka"}[5m])) != 0 - for: 5m - labels: - severity: warning - - alert: KafkaISRShrinkRate - annotations: - description: Kafka instance {{ $labels.instance }} in cluster {{ $labels.kafka_cluster - }} ISR is shrinking by {{ $value }} per second. If a broker goes down, ISR - for some of the partitions shrink. When that broker is up again, ISRs are - expanded once the replicas are fully caught up. Other than that, the expected - value for ISR shrink rate is 0. If ISR is expanding and shrinking frequently, - adjust Allowed replica lag. - summary: Kafka ISR Expansion Rate is shrinking. - expr: | - sum without() (rate(kafka_server_replicamanager_isrshrinkspersec{job="integrations/kafka"}[5m])) != 0 - for: 5m - labels: - severity: warning - - alert: KafkaBrokerCount - annotations: - description: Kafka cluster {{ $labels.kafka_cluster }} broker count is 0. - summary: Kafka has no Brokers online. - expr: count without(instance) (kafka_server_kafkaserver_brokerstate{job="integrations/kafka"}) - == 0 - for: 5m - labels: - severity: critical - - alert: KafkaZookeeperSyncConnect - annotations: - description: Kafka instance {{ $labels.instance }} in cluster {{ $labels.kafka_cluster - }} Zookeeper Sync Disconected. - summary: Kafka Zookeeper Sync Disconected. - expr: | - avg without() (kafka_server_sessionexpirelistener_zookeepersyncconnectspersec{job="integrations/kafka"}) < 0 - for: 5m - labels: - severity: warning +null diff --git a/assets/kafka/dashboards/connect-overview.json b/assets/kafka/dashboards/connect-overview.json deleted file mode 100644 index ae07849..0000000 --- a/assets/kafka/dashboards/connect-overview.json +++ /dev/null @@ -1,6265 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Monitor Apache Kafka Connect", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 8, - "links": [ ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 199, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "General", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "decimals": 0, - "mappings": [ ], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ ] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 212, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_total_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Tasks Total", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "decimals": 0, - "mappings": [ ], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ ] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 213, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_running_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Tasks Running", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "decimals": 0, - "mappings": [ ], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 1 - } - ] - } - }, - "overrides": [ ] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 1 - }, - "id": 215, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_paused_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Tasks Paused", - "transformations": [ ], - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "decimals": 0, - "mappings": [ ], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [ ] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 1 - }, - "id": 214, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_failed_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Tasks Failed", - "transformations": [ ], - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "decimals": 0, - "mappings": [ ], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - } - ] - } - }, - "overrides": [ ] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 1 - }, - "id": 216, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_unassigned_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Tasks Unassigned", - "transformations": [ ], - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "decimals": 0, - "mappings": [ ], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "purple", - "value": 1 - } - ] - } - }, - "overrides": [ ] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 217, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_destroyed_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Tasks Destroyed", - "transformations": [ ], - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [ ] - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 4 - }, - "id": 227, - "links": [ ], - "maxDataPoints": 1, - "options": { - "displayLabels": [ ], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true, - "values": [ ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "7.0.5", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum (kafka_connect_connector_metrics{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",status=\"running\"})", - "instant": true, - "interval": "", - "legendFormat": "running", - "refId": "B" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum (kafka_connect_connector_metrics{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",status=\"stopped\"})", - "instant": true, - "interval": "", - "legendFormat": "stopped", - "refId": "A" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum (kafka_connect_connector_metrics{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",status=\"paused\"})", - "instant": true, - "interval": "", - "legendFormat": "paused", - "refId": "C" - } - ], - "title": "Connector repartition per status", - "type": "piechart" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [ ] - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 4 - }, - "id": 219, - "links": [ ], - "maxDataPoints": 1, - "options": { - "displayLabels": [ ], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true, - "values": [ ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "7.0.5", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_running_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "instant": true, - "interval": "", - "legendFormat": "running", - "refId": "A" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_failed_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "instant": true, - "interval": "", - "legendFormat": "failed", - "refId": "B" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_paused_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "instant": true, - "interval": "", - "legendFormat": "paused", - "refId": "C" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_unassigned_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "instant": true, - "interval": "", - "legendFormat": "unassigned", - "refId": "D" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_destroyed_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "instant": true, - "interval": "", - "legendFormat": "destroyed", - "refId": "E" - } - ], - "title": "Task repartition per status", - "type": "piechart" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Status of connectors over time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "paused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/stopped.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2495C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/paused.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF9830", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/running.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#73BF69", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 228, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum (kafka_connect_connector_metrics{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",status!=\"\"}) by (status) ", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "title": "Status of connectors", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Status of tasks over time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "destroyed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "failed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "paused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "running" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "unassigned" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsZero", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 226, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_running_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "running", - "refId": "A" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_failed_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "failed", - "refId": "B" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_paused_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "paused", - "refId": "C" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_destroyed_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "destroyed", - "refId": "D" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_connect_connect_worker_metrics_connector_unassigned_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "unassigned", - "refId": "E" - } - ], - "title": "Status of tasks", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 221, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "System", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Cores", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 18 - }, - "id": 223, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "rate(process_cpu_seconds_total{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{cluster}}/{{instance}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Memory", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 18 - }, - "id": 224, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(area)(jvm_memory_bytes_used{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "{{cluster}}/{{instance}}", - "refId": "A" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "jvm_memory_bytes_max{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",area=\"heap\"}", - "interval": "", - "legendFormat": "{{cluster}}/{{instance}}", - "refId": "B" - } - ], - "title": "JVM Memory Used", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "% time in GC", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 18 - }, - "id": 225, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{cluster}}/{{instance}}", - "refId": "A" - } - ], - "title": "JVM Memory Used", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 97, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Connect Worker", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "right", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "minWidth": 50 - }, - "decimals": 2, - "displayName": "", - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "env" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "client_id" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "start_time_ms" - }, - "properties": [ - { - "id": "displayName", - "value": "Startup time" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "unit", - "value": "time: MMMM D, YYYY LT" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #C" - }, - "properties": [ - { - "id": "displayName", - "value": "Connector Count" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #D" - }, - "properties": [ - { - "id": "displayName", - "value": "Connector Startup Success Total" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #E" - }, - "properties": [ - { - "id": "displayName", - "value": "Connector Startup Failure Total" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #F" - }, - "properties": [ - { - "id": "displayName", - "value": "Number of rebalances" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #G" - }, - "properties": [ - { - "id": "displayName", - "value": "Average time of Rebalances" - }, - { - "id": "unit", - "value": "ms" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #H" - }, - "properties": [ - { - "id": "displayName", - "value": "Time since last rebalance" - }, - { - "id": "unit", - "value": "ms" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "displayName", - "value": "Worker instance" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #I" - }, - "properties": [ - { - "id": "displayName", - "value": "Number of tasks" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #J" - }, - "properties": [ - { - "id": "displayName", - "value": "Task Startup Success " - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #K" - }, - "properties": [ - { - "id": "displayName", - "value": "Task Startup Failure" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.align" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 146, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_app_info{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",client_id!=\"\",start_time_ms!=\"\"}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "B" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_app_info{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",client_id!=\"\",version!=\"\"}", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum by (instance) (kafka_connect_connect_worker_metrics_connector_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "C" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum by (instance) (kafka_connect_connect_worker_metrics_connector_startup_success_total{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "D" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum by (instance) (kafka_connect_connect_worker_metrics_connector_startup_failure_total{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "E" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum by (instance) (kafka_connect_connect_worker_metrics_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "I" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum by (instance) (kafka_connect_connect_worker_metrics_task_startup_success_total{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "J" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum by (instance) (kafka_connect_connect_worker_metrics_task_startup_failure_total{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "K" - } - ], - "title": "Connect Worker", - "transformations": [ - { - "id": "merge", - "options": { - "reducers": [ ] - } - } - ], - "transparent": true, - "type": "table" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Average number of network operations (reads or writes) on all connections per second", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 35 - }, - "id": 95, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connect_metrics_network_io_rate{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",client_id!=\"\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}/{{client_id}}", - "refId": "A" - } - ], - "title": "Network IO Rate", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Bytes per second read off all sockets", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 35 - }, - "id": 91, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connect_metrics_incoming_byte_rate{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",client_id!=\"\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}/{{client_id}}", - "refId": "A" - } - ], - "title": "Incoming Byte Rate", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Average number of outgoing bytes sent per second to all servers", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 35 - }, - "id": 171, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connect_metrics_outgoing_byte_rate{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",client_id!=\"\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}/{{client_id}}", - "refId": "A" - } - ], - "title": "Outgoing Byte Rate", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Current number of active connections", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "connections" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 42 - }, - "id": 169, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connect_metrics_connection_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",client_id!=\"\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}/{{client_id}}", - "refId": "A" - } - ], - "title": "Current number of active connections", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Connections that failed authentication", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "connections" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 42 - }, - "id": 170, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connect_metrics_failed_authentication_total{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",client_id!=\"\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}/{{client_id}}", - "refId": "A" - } - ], - "title": "Failed authentication connections", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Connections that were successfully authenticated using SASL or SSL", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "connections" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 42 - }, - "id": 174, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connect_metrics_successful_authentication_rate{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",client_id!=\"\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}/{{client_id}}", - "refId": "A" - } - ], - "title": "Success authentication connections", - "type": "timeseries" - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${datasource}" - }, - "description": "Average number of requests sent per second", - "fieldConfig": { - "defaults": { - "links": [ ] - }, - "overrides": [ ] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 49 - }, - "hiddenSeries": false, - "id": 172, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.2.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connect_metrics_request_rate{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",client_id!=\"\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}/{{client_id}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeRegions": [ ], - "title": "Average number of requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${datasource}" - }, - "description": "Responses received and sent per second", - "fieldConfig": { - "defaults": { - "links": [ ] - }, - "overrides": [ ] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 49 - }, - "hiddenSeries": false, - "id": 173, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.2.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connect_metrics_response_rate{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",client_id!=\"\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}/{{client_id}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeRegions": [ ], - "title": "Responses received and sent", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "reqps", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${datasource}" - }, - "description": "Fraction of time the I/O thread spent doing I/O", - "fieldConfig": { - "defaults": { - "links": [ ] - }, - "overrides": [ ] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 49 - }, - "hiddenSeries": false, - "id": 93, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.2.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connect_metrics_io_ratio{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",client_id!=\"\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}/{{client_id}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeRegions": [ ], - "title": "IO Ratio", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 56 - }, - "id": 132, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Connector details", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "right", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "decimals": 2, - "displayName": "", - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "connector_class" - }, - "properties": [ - { - "id": "displayName", - "value": "class" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align", - "value": "left" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "env" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "displayName", - "value": "Nb of Tasks destroyed" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "#B877D9", - "value": 1 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #C" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #D" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "status" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 2 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "connector" - }, - "properties": [ - { - "id": "displayName", - "value": "name" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align", - "value": "left" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "connector_type" - }, - "properties": [ - { - "id": "displayName", - "value": "type" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align", - "value": "left" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "connector_version" - }, - "properties": [ - { - "id": "displayName", - "value": "version" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align", - "value": "left" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #E" - }, - "properties": [ - { - "id": "displayName", - "value": "Nb of tasks" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 1 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #F" - }, - "properties": [ - { - "id": "displayName", - "value": "Nb of Tasks running" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 1 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #G" - }, - "properties": [ - { - "id": "displayName", - "value": "Nb of Tasks failed" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "#F2495C", - "value": 1 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #H" - }, - "properties": [ - { - "id": "displayName", - "value": "Nb of Tasks paused" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "#FF9830", - "value": 1 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #I" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #J" - }, - "properties": [ - { - "id": "displayName", - "value": "Nb of Tasks unassigned" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "#FADE2A", - "value": 1 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 57 - }, - "id": 129, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "label_replace(label_replace(label_replace(kafka_connect_connector_info{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",status!=\"\"}, \"status\", \"1\", \"status\", \"running\"), \"status\", \"2\", \"status\", \"paused\"), \"status\", \"3\", \"status\", \"stopped\")", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "I" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connector_info{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",connector_type!=\"\"}", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connector_info{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",connector_version!=\"\"}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "C" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connector_info{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",connector_class!=\"\"}", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "D" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum by (connector) (kafka_connect_connect_worker_metrics_connector_total_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "E" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum by (connector) (kafka_connect_connect_worker_metrics_connector_running_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "F" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum by (connector) (kafka_connect_connect_worker_metrics_connector_failed_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "G" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum by (connector) (kafka_connect_connect_worker_metrics_connector_paused_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "H" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum by (connector) (kafka_connect_connect_worker_metrics_connector_destroyed_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "B" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum by (connector) (kafka_connect_connect_worker_metrics_connector_unassigned_task_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"})", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "J" - } - ], - "title": "Connectors", - "transformations": [ - { - "id": "seriesToRows", - "options": { - "reducers": [ ] - } - } - ], - "type": "table" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 68 - }, - "id": 234, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Rebalances", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Rebalances average time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 69 - }, - "id": 209, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connect_worker_rebalance_metrics_rebalance_avg_time_ms{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{cluster}}/{{instance}}", - "refId": "A" - } - ], - "title": "Rebalances average time", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Time since last rebalance", - "fieldConfig": { - "defaults": { - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - } - ] - }, - "unit": "clockms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 77 - }, - "id": 230, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "repeat": "instance", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connect_worker_rebalance_metrics_time_since_last_rebalance_ms{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"} >= 0", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "($instance) Time since last rebalance ", - "type": "stat" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 81 - }, - "id": 112, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Task metrics", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Average size of the batches processed by the connector", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 82 - }, - "id": 113, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connector_task_metrics_batch_size_avg{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Batch Size Average", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Maximum size of the batches processed by the connector", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 82 - }, - "id": 114, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connector_task_metrics_batch_size_max{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Batch Size Max", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Average percentage of the task’s offset commit attempts that succeeded", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 89 - }, - "id": 115, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connector_task_metrics_offset_commit_success_percentage{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Offset commit success percentage", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "The average time in milliseconds taken by this task to commit offsets", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 89 - }, - "id": 116, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connector_task_metrics_offset_commit_avg_time_ms{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Offset commit Average Time", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "The fraction of time this task has spent in the running state.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 96 - }, - "id": 117, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_connector_task_metrics_running_ratio{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Running ratio", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 103 - }, - "id": 201, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Task Errors metrics", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Total number of failures seen by task", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 104 - }, - "id": 203, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_task_error_metrics_total_record_failures{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Total record failures", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Total number of errors seen by task", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 104 - }, - "id": 205, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_task_error_metrics_total_record_errors{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Total record errors", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Total number of records skipped by task", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 104 - }, - "id": 206, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_task_error_metrics_total_records_skipped{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Total record skipped", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "The number of messages that was logged into either the dead letter queue or with Log4j", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 111 - }, - "id": 208, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_task_error_metrics_total_errors_logged{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Total errors logged", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Total number of retries made by task", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 111 - }, - "id": 207, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_task_error_metrics_total_retries{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Total retries", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Number of produce requests to the dead letter queue", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 118 - }, - "id": 202, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_task_error_metrics_deadletterqueue_produce_requests{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Dead letter queue Produce requests", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Number of produce requests to the dead letter queue", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 118 - }, - "id": 204, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_task_error_metrics_deadletterqueue_produce_requests{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Dead letter queue Produce requests", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 125 - }, - "id": 139, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Source metrics", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "The average time in milliseconds taken by this task to poll for a batch of source records", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 126 - }, - "id": 140, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_source_task_metrics_poll_batch_avg_time_ms{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",task!=\"\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Poll Batch Average time", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "The maximum time in milliseconds taken by this task to poll for a batch of source records", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 126 - }, - "id": 141, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_source_task_metrics_poll_batch_max_time_ms{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",task!=\"\"}", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Poll Batch Max time", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "The average per-second number of records produced/polled (before transformation) by this task belonging to the named source connector in this worker.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 133 - }, - "id": 144, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_source_task_metrics_source_record_poll_rate{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",task!=\"\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Source Record Poll rate", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "The average per-second number of records output from the transformations and written to Kafka for this task belonging to the named source connector in this worker. This is after transformations are applied and excludes any records filtered out by the transformations.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 133 - }, - "id": 143, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_source_task_metrics_source_record_write_rate{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",task!=\"\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Source Record Write rate", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "The average number of records that have been produced by this task but not yet completely written to Kafka.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 140 - }, - "id": 142, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_source_task_metrics_source_record_active_count_avg{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",task!=\"\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Source Record Active Count average", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "The maximum number of records that have been produced by this task but not yet completely written to Kafka.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 140 - }, - "id": 145, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_source_task_metrics_source_record_active_count_max{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",task!=\"\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Source Record Active Count max", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 147 - }, - "id": 134, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Sink metrics", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "The number of topic partitions assigned to this task belonging to the named sink connector in this worker.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 148 - }, - "id": 135, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_sink_task_metrics_partition_count{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",task!=\"\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Partition Count", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "The average time in milliseconds taken by this task to put a batch of sinks records", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 148 - }, - "id": 136, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_sink_task_metrics_put_batch_avg_time_ms{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",task!=\"\"}", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Put Batch Average time", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "The maximum time in milliseconds taken by this task to put a batch of sinks records", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 148 - }, - "id": 137, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_connect_sink_task_metrics_put_batch_max_time_ms{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",connector=~\"$connector\",task!=\"\"}", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{connector}}-{{task}}", - "refId": "A" - } - ], - "title": "Put Batch Max time", - "type": "timeseries" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "kafka-integration" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "label": "Data source", - "multi": false, - "name": "datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "options": [ ], - "query": "label_values(kafka_connect_app_info{job=~\"integrations/kafka-connect|integrations/kafka\"}, job)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "$datasource" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Cluster", - "multi": true, - "name": "cluster", - "options": [ ], - "query": "label_values(kafka_server_kafkaserver_brokerstate{job=~\"$job\"}, cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Kafka_cluster", - "multi": true, - "name": "kafka_cluster", - "options": [ ], - "query": "label_values(kafka_connect_app_info{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\"}, kafka_cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Instance", - "multi": true, - "name": "instance", - "options": [ ], - "query": "label_values(kafka_connect_app_info{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${datasource}" - }, - "definition": "label_values(kafka_connect_connector_task_metrics_pause_ratio{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"},connector)", - "hide": 0, - "includeAll": true, - "label": "Connector", - "multi": true, - "name": "connector", - "options": [ ], - "query": "label_values(kafka_connect_connector_task_metrics_pause_ratio{job=~\"integrations/kafka-connect|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"},connector)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "hidden": false, - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kafka Connect Overview", - "uid": "AEaSQ97mz", - "version": 2, - "weekStart": "" -} diff --git a/assets/kafka/dashboards/kafka-ksqldb-overview.json b/assets/kafka/dashboards/kafka-ksqldb-overview.json deleted file mode 100644 index e5e82c1..0000000 --- a/assets/kafka/dashboards/kafka-ksqldb-overview.json +++ /dev/null @@ -1,4419 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 5, - "links": [ ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 29, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Overview", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Average number of active queries per server.", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#d44a3a", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0 - }, - { - "color": "#299c46", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 18, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "avg(ksql_ksql_engine_query_stats_num_active_queries{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Active Queries", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Num of created queries", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 500 - }, - { - "color": "#d44a3a", - "value": 800 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 20, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "avg(ksql_ksql_engine_query_stats_running_queries{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Running Queries", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Num of persisted queries", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#d44a3a", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0 - }, - { - "color": "#299c46", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "id": 2, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "avg(ksql_ksql_engine_query_stats_num_persistent_queries{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Persisted Queries", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Num of rebalancing queries", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 1 - }, - { - "color": "#d44a3a", - "value": 5 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "id": 16, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(ksql_ksql_engine_query_stats_rebalancing_queries{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Rebalancing Queries", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Number of error query", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 1 - }, - { - "color": "#d44a3a", - "value": 2 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "id": 4, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "avg(ksql_ksql_engine_query_stats_error_queries{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Queries in Error State", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Num of idle queries", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 1 - }, - { - "color": "#d44a3a", - "value": 5 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 19, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(ksql_ksql_engine_query_stats_num_idle_queries{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Idle Queries", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "cellOptions": { - "type": "auto" - }, - "filterable": false, - "inspect": false - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "ksql_query" - }, - "properties": [ - { - "id": "custom.width", - "value": 426 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "custom.width", - "value": 381 - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 16, - "x": 0, - "y": 5 - }, - "id": 23, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ ] - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "ksql_ksql_metrics_ksql_queries_query_status{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Queries Status", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value": true, - "__name__": true, - "env": true, - "job": true, - "ksql_cluster": true - }, - "indexByName": { }, - "renameByName": { - "Time": "", - "__name__": "", - "instance": "", - "ksql_cluster": "", - "ksql_query": "" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Num of not running queries", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 1 - }, - { - "color": "#d44a3a", - "value": 5 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 5 - }, - "id": 5, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(ksql_ksql_engine_query_stats_not_running_queries{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Stopped Queries", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Num of running queries", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 1 - }, - { - "color": "#d44a3a", - "value": 5 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 5 - }, - "id": 15, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(ksql_ksql_engine_query_stats_pending_shutdown_queries{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Currently Shutting Down Queries", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Cluster liveness", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 9 - }, - "id": 17, - "links": [ ], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "ksql_ksql_engine_query_stats_liveness_indicator{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Cluster liveness", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Message consumed/sec", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 14 - }, - "id": 21, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "ksql_ksql_engine_query_stats_messages_consumed_per_sec{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Message consumed/sec", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Message produced/sec", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "irate(ksql_ksql_engine_query_stats_messages_produced_per_sec{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Message produced/sec", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 33, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "System", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 23 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "irate(process_cpu_seconds_total{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 23 - }, - "id": 24, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(area)(jvm_memory_bytes_used{ job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "Used:{{instance}}", - "refId": "A" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "jvm_memory_bytes_max{area=\"heap\", job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "Max:{{instance}}", - "refId": "B" - } - ], - "title": "JVM Memory Used", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 4, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 23 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Time spent in GC", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 31, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Queries Performance", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 0, - "y": 33 - }, - "id": 26, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_thread_metrics_poll_latency_avg{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{thread_id}}_avg", - "refId": "A" - } - ], - "title": "Poll Latency (Avg)", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 6, - "y": 33 - }, - "id": 35, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_thread_metrics_poll_latency_max{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}_max", - "refId": "B" - } - ], - "title": "Poll Latency (Max)", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 12, - "y": 33 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_thread_metrics_process_latency_avg{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{thread_id}}_avg", - "refId": "A" - } - ], - "title": "Process Latency (Avg)", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 18, - "y": 33 - }, - "id": 34, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_thread_metrics_process_latency_max{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}_max", - "refId": "B" - } - ], - "title": "Process Latency Max", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 0, - "y": 44 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_thread_metrics_commit_latency_avg{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{thread_id}}_avg", - "refId": "A" - } - ], - "title": "Commit Latency (Avg)", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 6, - "y": 44 - }, - "id": 38, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_thread_metrics_commit_latency_max{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{thread_id}}_avg", - "refId": "A" - } - ], - "title": "Commit Latency (Max)", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 12, - "y": 44 - }, - "id": 27, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_thread_metrics_punctuate_latency_avg{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{thread_id}}_avg", - "refId": "A" - } - ], - "title": "Punctuate Latency (Avg)", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/max/" - }, - "properties": [ - { - "id": "custom.lineStyle", - "value": { - "dash": [ - 5, - 2 - ], - "fill": "dash" - } - } - ] - }, - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 18, - "y": 44 - }, - "id": 37, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_thread_metrics_punctuate_latency_max{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}_max", - "refId": "B" - } - ], - "title": "Punctuate Latency (Max)", - "transformations": [ ], - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 55 - }, - "id": 40, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "StateStore Metric", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 0, - "y": 56 - }, - "id": 36, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_state_metrics_put_rate{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Put Rate", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 9, - "x": 6, - "y": 56 - }, - "id": 42, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_state_metrics_put_latency_avg{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Put average latency", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 9, - "x": 15, - "y": 56 - }, - "id": 43, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_state_metrics_put_latency_max{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Put max latency", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 0, - "y": 67 - }, - "id": 52, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_state_metrics_put_if_absent_rate_rate{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Put if absent rate", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 9, - "x": 6, - "y": 67 - }, - "id": 53, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_state_metrics_put_if_absent_latency_avg{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Put if absent average latency", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 9, - "x": 15, - "y": 67 - }, - "id": 54, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_state_metrics_put_if_absent_latency_max{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Put if absent max latency", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 0, - "y": 78 - }, - "id": 41, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_state_metrics_fetch_rate{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Fetch Rate", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 9, - "x": 6, - "y": 78 - }, - "id": 44, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_streams_stream_state_metrics_fetch_latency_avg{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Fetch average latency", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 9, - "x": 15, - "y": 78 - }, - "id": 45, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_state_metrics_put_latency_max{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Fetch max latency", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 0, - "y": 89 - }, - "id": 46, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_state_metrics_delete_rate{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Delete Rate", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 9, - "x": 6, - "y": 89 - }, - "id": 47, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_state_metrics_delete_latency_avg{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Delete average latency", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 9, - "x": 15, - "y": 89 - }, - "id": 48, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_state_metrics_delete_latency_max{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Delete max latency", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 0, - "y": 100 - }, - "id": 49, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_state_metrics_restore_rate{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Restore Rate", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 9, - "x": 6, - "y": 100 - }, - "id": 50, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_state_metrics_restore_latency_avg{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Restore average latency", - "transformations": [ ], - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 9, - "x": 15, - "y": 100 - }, - "id": 51, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_streams_stream_state_metrics_restore_latency_max{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{thread_id}}", - "refId": "B" - } - ], - "title": "Restore max latency", - "transformations": [ ], - "type": "timeseries" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "kafka-integration" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "label": "Data source", - "multi": false, - "name": "datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "options": [ ], - "query": "label_values(ksql_ksql_engine_query_stats_liveness_indicator{job=~\"integrations/kafka-ksqldb|integrations/kafka\"}, job)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "$datasource" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Cluster", - "multi": true, - "name": "cluster", - "options": [ ], - "query": "label_values(kafka_server_kafkaserver_brokerstate{job=~\"$job\"}, cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Kafka_cluster", - "multi": true, - "name": "kafka_cluster", - "options": [ ], - "query": "label_values(ksql_ksql_engine_query_stats_liveness_indicator{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\"}, kafka_cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Instance", - "multi": true, - "name": "instance", - "options": [ ], - "query": "label_values(ksql_ksql_engine_query_stats_liveness_indicator{job=~\"integrations/kafka-ksqldb|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "ksqldb Overview", - "uid": "pbx34foGk", - "version": 2, - "weekStart": "" -} diff --git a/assets/kafka/dashboards/kafka-lag-overview.json b/assets/kafka/dashboards/kafka-lag-overview.json deleted file mode 100644 index 92cb050..0000000 --- a/assets/kafka/dashboards/kafka-lag-overview.json +++ /dev/null @@ -1,833 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [ ], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "Kafka lag overview", - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 7589, - "graphTooltip": 0, - "id": 52, - "links": [ ], - "liveNow": false, - "panels": [ - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 14, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 480 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.7", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "sum(rate(kafka_topic_partition_current_offset{job=\"integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\", topic=~\"$topic\"}[$__rate_interval])) by (topic)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{topic}}", - "refId": "B" - } - ], - "title": "Message in per second", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 16, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 480 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.7", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "sum(increase(kafka_topic_partition_current_offset{job=\"integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\", topic=~\"$topic\"}[5m])/5) by (topic)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "title": "Message in per minute", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 20, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.7", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(rate(kafka_consumergroup_current_offset{job=\"integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\", topic=~\"$topic\"}[$__rate_interval])) by (consumergroup, topic)", - "interval": "", - "legendFormat": "{{consumergroup}} (topic: {{topic}})", - "refId": "A" - } - ], - "title": "Message consume per second", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 18, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 480 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.7", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "sum(increase(kafka_consumergroup_current_offset{job=\"integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\", topic=~\"$topic\"}[5m])/5) by (consumergroup, topic)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{consumergroup}} (topic: {{topic}})", - "refId": "A" - } - ], - "title": "Message consume per minute", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 20 - }, - "id": 22, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.7", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "exemplar": true, - "expr": "avg(kafka_consumer_lag_millis{job=\"integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\", topic=~\"$topic\"}/1000) by (consumergroup, topic)", - "interval": "", - "legendFormat": "{{consumergroup}} (topic: {{topic}})", - "refId": "A" - } - ], - "title": "Lag by consumer group in seconds", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 20 - }, - "id": 12, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 480 - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.7", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "avg(kafka_consumergroup_uncommitted_offsets{job=\"integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\", topic=~\"$topic\"}) by (consumergroup, topic)", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{consumergroup}} (topic: {{topic}})", - "refId": "A" - } - ], - "title": "Lag by consumer group", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ ] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 8, - "links": [ ], - "options": { - "barRadius": 0, - "barWidth": 0.84999999999999998, - "groupWidth": 0.69999999999999996, - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "9.2.7", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum by(topic) (kafka_topic_partitions{job=\"integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\", topic=~\"$topic\"})", - "format": "table", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "title": "Partitions per topic", - "type": "barchart" - } - ], - "refresh": "30s", - "schemaVersion": 37, - "style": "dark", - "tags": [ - "kafka-integration" - ], - "templating": { - "list": [ - { - "label": "Data source", - "name": "datasource", - "query": "prometheus", - "type": "datasource" - }, - { - "allValue": ".+", - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "query": "label_values(kafka_topic_partition_current_offset{job=\"integrations/kafka\"}, job)", - "refresh": 2, - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "includeAll": true, - "label": "Kafka_cluster", - "multi": true, - "name": "kafka_cluster", - "query": "label_values(kafka_topic_partition_current_offset{job=\"integrations/kafka\",job=~\"$job\"}, kafka_cluster)", - "refresh": 2, - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "includeAll": true, - "label": "Instance", - "multi": true, - "name": "instance", - "query": "label_values(kafka_topic_partition_current_offset{job=\"integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\"}, instance)", - "refresh": 2, - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${datasource}" - }, - "definition": "label_values(kafka_topic_partition_current_offset{job=\"integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\", topic!='__consumer_offsets',topic!='--kafka'}, topic)", - "hide": 0, - "includeAll": true, - "label": "Topic", - "multi": true, - "name": "topic", - "options": [ ], - "query": { - "query": "label_values(kafka_topic_partition_current_offset{job=\"integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\", topic!='__consumer_offsets',topic!='--kafka'}, topic)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Kafka lag overview", - "uid": "jwPKIsniz", - "version": 6 -} diff --git a/assets/kafka/dashboards/kafka-overview.json b/assets/kafka/dashboards/kafka-overview.json deleted file mode 100644 index 868a00a..0000000 --- a/assets/kafka/dashboards/kafka-overview.json +++ /dev/null @@ -1,7171 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Kafka resource usage and throughput", - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 721, - "graphTooltip": 0, - "id": 3, - "links": [ ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 42, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Healthcheck", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Number of active controllers in the cluster.", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "#e5ac0e", - "value": 2 - }, - { - "color": "#bf1b00" - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 12, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_controller_kafkacontroller_activecontrollercount{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Active Controllers", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Number of Brokers Online", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#d44a3a", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0 - }, - { - "color": "semi-dark-green", - "value": 2 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 14, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "repeatDirection": "h", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "count(kafka_server_replicamanager_leadercount{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Brokers Online", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Unclean leader election rate", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 1 - }, - { - "color": "#d44a3a" - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "id": 16, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Unclean Leader Election Rate", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 2 - }, - { - "color": "#d44a3a" - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "id": 33, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_controller_kafkacontroller_preferredreplicaimbalancecount{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Preferred Replica Imbalance", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Bytes/s", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 1 - }, - "id": 84, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic!=\"\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Bytes in", - "metric": "kafka_server_brokertopicmetrics_bytesinpersec", - "refId": "A", - "step": 4 - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic!=\"\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "Bytes out", - "metric": "kafka_server_brokertopicmetrics_bytesinpersec", - "refId": "B", - "step": 4 - } - ], - "title": "Broker network throughput", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Number of partitions that dont have an active leader and are hence not writable or readable.", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "#bf1b00", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 5 - }, - "id": 22, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Offline Partitions Count", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Number of under-replicated partitions (| ISR | < | all replicas |).", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 1 - }, - { - "color": "#bf1b00", - "value": 5 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 5 - }, - "id": 20, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Under Replicated Partitions", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Number of partitions under min insync replicas.", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "#bf1b00", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 5 - }, - "id": 32, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_cluster_partition_underminisr{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Under Min ISR Partitions", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Partitions that are online", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#d44a3a", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0 - }, - { - "color": "#299c46", - "value": 0 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 5 - }, - "id": 18, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Online Partitions", - "type": "stat" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 40, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "System", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Cores", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "localhost:7071" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 27, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "irate(process_cpu_seconds_total{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval])*100", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "process_cpu_secondspersec", - "refId": "A", - "step": 4 - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Memory", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "localhost:7071" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 2, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(area)(jvm_memory_bytes_used{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "jvm_memory_bytes_used", - "refId": "A", - "step": 4 - } - ], - "title": "JVM Memory Used", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "% time in GC", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "localhost:7071" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 3, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval]))", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "jvm_gc_collection_seconds_sum", - "refId": "A", - "step": 4 - } - ], - "title": "Time spent in GC", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 29, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Throughput In/Out", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Messages/s", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 18 - }, - "id": 4, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(instance)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}[$__rate_interval]))", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{topic}}", - "metric": "kafka_server_brokertopicmetrics_messagesinpersec", - "refId": "A", - "step": 4 - } - ], - "title": "Messages In Per Topic", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Bytes/s", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 18 - }, - "id": 5, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(instance)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}[$__rate_interval]))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{topic}}", - "metric": "kafka_server_brokertopicmetrics_bytesinpersec", - "refId": "A", - "step": 4 - } - ], - "title": "Bytes In Per Topic", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Bytes/s", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 18 - }, - "id": 6, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(instance)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}[$__rate_interval]))", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{topic}}", - "metric": "kafka_server_brokertopicmetrics_bytesinpersec", - "refId": "A", - "step": 4 - } - ], - "title": "Bytes Out Per Topic", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Messages/s", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 25 - }, - "id": 10, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}[$__rate_interval]))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "kafka_server_brokertopicmetrics_messagesinpersec", - "refId": "A", - "step": 4 - } - ], - "title": "Messages In Per Broker", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Bytes/s", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 25 - }, - "id": 7, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}[$__rate_interval]))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "kafka_server_brokertopicmetrics_bytesinpersec", - "refId": "A", - "step": 4 - } - ], - "title": "Bytes In Per Broker", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 25 - }, - "id": 9, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}[$__rate_interval]))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Bytes Out Per Broker", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 117, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Replication", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Offline partitions over time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 122, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "Online Partitions", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Offline partitions over time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 33 - }, - "id": 121, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Offline Partitions", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Under replicated partitions over time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 41 - }, - "id": 120, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_cluster_partition_underreplicated{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Under Replicated Partitions", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Under min in sync replicas partitions over time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 41 - }, - "id": 119, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_cluster_partition_underminisr{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Under Min ISR Partitions", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 49 - }, - "id": 44, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Thread utilization", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available) ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 50 - }, - "id": 24, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_socketserver_networkprocessoravgidlepercent{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Network Processor Avg Idle Percent", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available). ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 50 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Request Handler Avg Idle Percent", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 126, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestchannel_requestqueuesize{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Request Queue Size", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 127, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestchannel_responsequeuesize{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",processor=\"\"}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Response Queue Size", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 66 - }, - "id": 86, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Zookeeper", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Latency in millseconds for ZooKeeper requests from broker. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 88, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_server_zookeeperclientmetrics_zookeeperrequestlatencyms{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "Zookeeper Request Latency", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 92, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "rate(kafka_server_sessionexpirelistener_zookeepersyncconnectspersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval])", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Zookeeper connections per sec", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 75 - }, - "id": 89, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "rate(kafka_server_sessionexpirelistener_zookeeperexpirespersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Zookeeper expired connections per sec", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 75 - }, - "id": 90, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "rate(kafka_server_sessionexpirelistener_zookeeperdisconnectspersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Zookeeper disconnect per sec", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 83 - }, - "id": 91, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "rate(kafka_server_sessionexpirelistener_zookeeperauthfailurespersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Zookeeper auth failures per sec", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 91 - }, - "id": 82, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Isr Shrinks / Expands", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 92 - }, - "id": 80, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_server_replicamanager_isrshrinkspersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "IsrShrinks per Sec", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 92 - }, - "id": 83, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_server_replicamanager_isrexpandspersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "hide": false, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "IsrExpands per Sec", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 100 - }, - "id": 53, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Logs size", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 101 - }, - "id": 55, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_log_log_size{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}) by (topic)", - "interval": "", - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "title": "Log size per Topic", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 101 - }, - "id": 56, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_log_log_size{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}) by (instance)", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Log size per Broker", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 109 - }, - "id": 58, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Producer Performance", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 110 - }, - "id": 60, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "Producer - RequestQueueTimeMs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 110 - }, - "id": 61, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_localtimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "Producer - LocalTimeMs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 118 - }, - "id": 62, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_remotetimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "Producer - RemoteTimeMs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 118 - }, - "id": 63, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "Producer - ResponseQueueTimeMs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 126 - }, - "id": 64, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "Producer - ResponseSendTimeMs", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 134 - }, - "id": 68, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Consumer Performance", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 135 - }, - "id": 69, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "Consumer - RequestQueueTimeMs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 135 - }, - "id": 70, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_localtimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "Consumer - LocalTimeMs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 143 - }, - "id": 71, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_remotetimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "Consumer - RemoteTimeMs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 143 - }, - "id": 72, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "Consumer - ResponseQueueTimeMs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 151 - }, - "id": 73, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "Consumer - ResponseSendTimeMs", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 159 - }, - "id": 66, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Fetch Follower Performance", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 160 - }, - "id": 74, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "FetchFollower - RequestQueueTimeMs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 160 - }, - "id": 75, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_localtimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "FetchFollower - LocalTimeMs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 168 - }, - "id": 76, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_remotetimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "FetchFollower - RemoteTimeMs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 168 - }, - "id": 77, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "FetchFollower - ResponseQueueTimeMs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 176 - }, - "id": 78, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } - ], - "title": "FetchFollower - ResponseSendTimeMs", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 184 - }, - "id": 97, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Group Coordinator", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Number of consumer groups per group coordinator", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 185 - }, - "id": 99, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_coordinator_group_groupmetadatamanager_numgroups{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "instant": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Consumer groups number per coordinator", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Number of consumer group per state", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 185 - }, - "id": 100, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsstable{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "instant": false, - "interval": "", - "legendFormat": "stable", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupspreparingrebalance{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "preparing-rebalance", - "refId": "B" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsdead{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "dead", - "refId": "C" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupscompletingrebalance{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "completing-rebalance", - "refId": "D" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsempty{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "empty", - "refId": "E" - } - ], - "title": "Nb consumer groups per state", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 193 - }, - "id": 102, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Connections", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 194 - }, - "id": 104, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}) by (listener)", - "interval": "", - "legendFormat": "{{listener}}", - "refId": "A" - } - ], - "title": "Connections count per listener", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 194 - }, - "id": 105, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}) by (instance)", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Connections count per broker", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 202 - }, - "id": 106, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}) by (listener)", - "interval": "", - "legendFormat": "{{listener}}", - "refId": "A" - } - ], - "title": "Connections creation rate per listener", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 202 - }, - "id": 107, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}) by (instance)", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Connections creation rate per instance", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 210 - }, - "id": 108, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}) by (listener)", - "interval": "", - "legendFormat": "{{listener}}", - "refId": "A" - } - ], - "title": "Connections close rate per listener", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 210 - }, - "id": 110, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}) by (instance)", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Connections close rate per instance", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Tracks the amount of time Acceptor is blocked from accepting connections. See KIP-402 for more details.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 218 - }, - "id": 124, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "kafka_network_acceptor_acceptorblockedpercent{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{instance}} - {{listener}}", - "refId": "A" - } - ], - "title": "Acceptor Blocked Percentage", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 218 - }, - "id": 113, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_server_socketservermetrics_connections{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}) by (client_software_name, client_software_version)", - "interval": "", - "legendFormat": "{{client_software_name}} {{client_software_version}}", - "refId": "A" - } - ], - "title": "Connections per client version", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 226 - }, - "id": 31, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Request rate", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Total request rate.", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 227 - }, - "id": 37, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Request Per Sec", - "type": "stat" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Produce request rate.", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 227 - }, - "id": 112, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",request=\"Produce\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Produce Request Per Sec", - "type": "stat" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Fetch request rate.", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 227 - }, - "id": 111, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",request=\"FetchConsumer\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Fetch Request Per Sec", - "type": "stat" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Offset Commit request rate.", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 227 - }, - "id": 38, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",request=\"OffsetCommit\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Offset Commit Request Per Sec", - "type": "stat" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Metadata request rate.", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 227 - }, - "id": 36, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",request=\"Metadata\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Metadata Request Per Sec", - "type": "stat" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 231 - }, - "id": 94, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(rate(kafka_server_brokertopicmetrics_totalproducerequestspersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}[$__rate_interval])) by (topic)", - "interval": "", - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "title": "Produce request per sec per topic", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 231 - }, - "id": 95, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(rate(kafka_server_brokertopicmetrics_totalfetchrequestspersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}[$__rate_interval])) by (topic)", - "interval": "", - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "title": "Fetch request per sec per topic", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "uid": "$datasource" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 239 - }, - "id": 46, - "panels": [ ], - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "refId": "A" - } - ], - "title": "Message Conversion", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "The number of messages produced converted to match the log.message.format.version.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 240 - }, - "id": 48, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(rate(kafka_server_brokertopicmetrics_producemessageconversionspersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Number of procuded message conversion", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "The number of messages consumed converted at consumer to match the log.message.format.version.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 240 - }, - "id": 51, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum(rate(kafka_server_brokertopicmetrics_fetchmessageconversionspersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "title": "Number of consumed message conversion", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Number of connection per client version", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [ ] - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 248 - }, - "id": 115, - "links": [ ], - "options": { - "displayLabels": [ ], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true, - "values": [ ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(kafka_server_socketservermetrics_connections{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}) by (client_software_name, client_software_version) ", - "interval": "", - "legendFormat": "{{client_software_name}} - {{client_software_version}}", - "refId": "A" - } - ], - "title": "Client version repartition", - "type": "piechart" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "kafka-integration" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "label": "Data source", - "multi": false, - "name": "datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "options": [ ], - "query": "label_values(kafka_server_kafkaserver_brokerstate{job=\"integrations/kafka\"}, job)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "$datasource" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Cluster", - "multi": true, - "name": "cluster", - "options": [ ], - "query": "label_values(kafka_server_kafkaserver_brokerstate{job=~\"$job\"}, cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Kafka_cluster", - "multi": true, - "name": "kafka_cluster", - "options": [ ], - "query": "label_values(kafka_server_kafkaserver_brokerstate{job=\"integrations/kafka\",job=~\"$job\"}, kafka_cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Instance", - "multi": true, - "name": "instance", - "options": [ ], - "query": "label_values(kafka_server_kafkaserver_brokerstate{job=\"integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": { - "selected": true, - "text": [ - "0.99" - ], - "value": [ - "0.99" - ] - }, - "datasource": { - "uid": "${datasource}" - }, - "definition": "label_values(kafka_network_requestmetrics_requestqueuetimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}, quantile)", - "hide": 0, - "includeAll": true, - "label": "Percentile", - "multi": true, - "name": "percentile", - "options": [ ], - "query": "label_values(kafka_network_requestmetrics_requestqueuetimems{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}, quantile)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${datasource}" - }, - "definition": "label_values(kafka_log_log_size{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"},topic)", - "hide": 0, - "includeAll": true, - "label": "Topic", - "multi": true, - "name": "topic", - "options": [ ], - "query": "label_values(kafka_log_log_size{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"},topic)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Kafka Overview", - "uid": "qu-QZdfZz", - "version": 7, - "weekStart": "" -} diff --git a/assets/kafka/dashboards/kafka-topics.json b/assets/kafka/dashboards/kafka-topics.json deleted file mode 100644 index 3cc38a7..0000000 --- a/assets/kafka/dashboards/kafka-topics.json +++ /dev/null @@ -1,750 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [ ], - "liveNow": false, - "panels": [ - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(instance) (rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "title": "Messages In", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(instance) (rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "title": "Bytes In", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 4, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(instance) (rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "title": "Bytes Out", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "cellOptions": { - "type": "auto" - }, - "filterable": false, - "inspect": false - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "partition" - }, - "properties": [ - { - "id": "custom.width", - "value": 103 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "offset" - }, - "properties": [ - { - "id": "custom.width", - "value": 226 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 6, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "partition" - } - ] - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_log_log_logstartoffset{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Start Offset", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true, - "env": true, - "instance": false, - "job": true - }, - "indexByName": { - "Time": 0, - "Value": 7, - "__name__": 1, - "env": 2, - "instance": 3, - "job": 4, - "partition": 6, - "topic": 5 - }, - "renameByName": { - "Value": "offset" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "cellOptions": { - "type": "auto" - }, - "filterable": false, - "inspect": false - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "partition" - }, - "properties": [ - { - "id": "custom.width", - "value": 103 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "offset" - }, - "properties": [ - { - "id": "custom.width", - "value": 226 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 7, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "partition" - } - ] - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_log_log_logendoffset{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",topic=~\"$topic\"}", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "End Offset", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true, - "env": true, - "instance": false, - "job": true - }, - "indexByName": { - "Time": 0, - "Value": 7, - "__name__": 1, - "env": 2, - "instance": 3, - "job": 4, - "partition": 6, - "topic": 5 - }, - "renameByName": { - "Value": "offset" - } - } - } - ], - "type": "table" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "kafka-integration" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "label": "Data source", - "multi": false, - "name": "datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "options": [ ], - "query": "label_values(kafka_log_log_size{job=\"integrations/kafka\"}, job)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "uid": "$datasource" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Cluster", - "multi": true, - "name": "cluster", - "options": [ ], - "query": "label_values(kafka_server_kafkaserver_brokerstate{job=~\"$job\"}, cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Kafka_cluster", - "multi": true, - "name": "kafka_cluster", - "options": [ ], - "query": "label_values(kafka_log_log_size{job=\"integrations/kafka\",job=~\"$job\"}, kafka_cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Instance", - "multi": true, - "name": "instance", - "options": [ ], - "query": "label_values(kafka_log_log_size{job=\"integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "${datasource}" - }, - "definition": "label_values(kafka_log_log_size{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"},topic)", - "hide": 0, - "includeAll": true, - "label": "Topic name", - "multi": true, - "name": "topic", - "options": [ ], - "query": "label_values(kafka_log_log_size{job=\"integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"},topic)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { }, - "timezone": "", - "title": "Kafka Topics", - "uid": "vQT4b1-Mz", - "version": 2, - "weekStart": "" -} diff --git a/assets/kafka/dashboards/schema-registry-overview.json b/assets/kafka/dashboards/schema-registry-overview.json deleted file mode 100644 index 0516994..0000000 --- a/assets/kafka/dashboards/schema-registry-overview.json +++ /dev/null @@ -1,1147 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 7, - "links": [ ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 19, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "refId": "A" - } - ], - "title": "Schemas", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ ] - }, - "gridPos": { - "h": 10, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "avg(kafka_schema_registry_registered_count{job=~\"integrations/kafka-schemaregistry|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Schemas registered", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 4, - "y": 1 - }, - "id": 11, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "avg by(schema_type) (kafka_schema_registry_schemas_created{job=~\"integrations/kafka-schemaregistry|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "{{schema_type}}", - "refId": "A" - } - ], - "title": "Schema registered over time", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [ ] - }, - "overrides": [ ] - }, - "gridPos": { - "h": 10, - "w": 4, - "x": 16, - "y": 1 - }, - "id": 8, - "links": [ ], - "options": { - "displayLabels": [ ], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true, - "values": [ ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "7.3.4", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "avg by(schema_type)(kafka_schema_registry_schemas_created{job=~\"integrations/kafka-schemaregistry|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "instant": true, - "interval": "", - "legendFormat": "{{schema_type}}", - "refId": "A" - } - ], - "title": "Schemas created", - "type": "piechart" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [ ] - }, - "overrides": [ ] - }, - "gridPos": { - "h": 10, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 9, - "links": [ ], - "options": { - "displayLabels": [ ], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true, - "values": [ ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "7.3.4", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "avg by(schema_type)(kafka_schema_registry_schemas_deleted{job=~\"integrations/kafka-schemaregistry|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "instant": true, - "interval": "", - "legendFormat": "{{schema_type}}", - "refId": "A" - } - ], - "title": "Schemas deleted", - "type": "piechart" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 15, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "refId": "A" - } - ], - "title": "System", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 12 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "irate(process_cpu_seconds_total{job=~\"integrations/kafka-schemaregistry|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval])*100", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 12 - }, - "id": 21, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(area)(jvm_memory_bytes_used{job=~\"integrations/kafka-schemaregistry|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "Used:{{instance}}", - "refId": "A" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "jvm_memory_bytes_max{job=~\"integrations/kafka-schemaregistry|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",area=\"heap\"}", - "interval": "", - "legendFormat": "Max:{{instance}}", - "refId": "B" - } - ], - "title": "JVM Memory Used", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 3, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 12 - }, - "id": 23, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=~\"integrations/kafka-schemaregistry|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Time spent in GC", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 17, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "refId": "A" - } - ], - "title": "Connections", - "type": "row" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 20 - }, - "id": 4, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_schema_registry_jetty_metrics_connections_active{job=~\"integrations/kafka-schemaregistry|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Active Connections", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 20 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_schema_registry_jersey_metrics_request_rate{job=~\"integrations/kafka-schemaregistry|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Requests Rate", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 20 - }, - "id": 24, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "kafka_schema_registry_jersey_metrics_request_latency_99{job=~\"integrations/kafka-schemaregistry|integrations/kafka\",job=~\"$job\",cluster=~\"$cluster\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Requests latency 99p", - "type": "timeseries" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [ - "kafka-integration" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "label": "Data source", - "multi": false, - "name": "datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "options": [ ], - "query": "label_values(kafka_schema_registry_registered_count{job=~\"integrations/kafka-schemaregistry|integrations/kafka\"}, job)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "uid": "$datasource" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Cluster", - "multi": true, - "name": "cluster", - "options": [ ], - "query": "label_values(kafka_server_kafkaserver_brokerstate{job=~\"$job\"}, cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Kafka_cluster", - "multi": true, - "name": "kafka_cluster", - "options": [ ], - "query": "label_values(kafka_schema_registry_registered_count{job=~\"integrations/kafka-schemaregistry|integrations/kafka\",job=~\"$job\"}, kafka_cluster)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Instance", - "multi": true, - "name": "instance", - "options": [ ], - "query": "label_values(kafka_schema_registry_registered_count{job=~\"integrations/kafka-schemaregistry|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { }, - "timezone": "", - "title": "Schema Registry Overview", - "uid": "9ixzve-Mk", - "version": 2, - "weekStart": "" -} diff --git a/assets/kafka/dashboards/zookeeper-overview.json b/assets/kafka/dashboards/zookeeper-overview.json deleted file mode 100644 index 438a4c7..0000000 --- a/assets/kafka/dashboards/zookeeper-overview.json +++ /dev/null @@ -1,1235 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [ ], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 125, - "links": [ ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 22, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "refId": "A" - } - ], - "title": "Health check", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Quorum size of zookeeper ensemble or number of nodes online for single instance deployments", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#d44a3a", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 2 - }, - { - "color": "#299c46", - "value": 3 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 2, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "# kafka operator case\ncount(zookeeper_quorumsize{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})\n# kafka grafana cloud integration case\nor count(zookeeper_status_quorumsize{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})\n# or single instance case (non-cluster)\nor count(up{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"} == 1)", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "Zookeeper nodes online", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Number of alive connections", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 100 - }, - { - "color": "#d44a3a", - "value": 200 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 4, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(zookeeper_numaliveconnections{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "Alive connections", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Number of queued requests in the server. This goes up when the server receives more requests than it can process", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 16, - "x": 8, - "y": 1 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "zookeeper_outstandingrequests{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{server_name}}/{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Outstanding requests", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 5 - }, - "id": 3, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "avg(zookeeper_inmemorydatatree_nodecount{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Number of ZNodes", - "type": "stat" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "description": "Number of watchers", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#299c46", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 500 - }, - { - "color": "#d44a3a", - "value": 1000 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 5 - }, - "id": 5, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "sum(zookeeper_inmemorydatatree_watchcount{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Number of watchers", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 20, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "refId": "A" - } - ], - "title": "System", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "# for strimzi operator: remove kafka container java metrics \nirate(process_cpu_seconds_total{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\", container!=\"kafka\"}[$__rate_interval])*100", - "interval": "", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum without(area)(jvm_memory_bytes_used{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\", container!=\"kafka\"})", - "interval": "", - "legendFormat": "Used:{{instance}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "jvm_memory_bytes_max{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\",area=\"heap\", container!=\"kafka\"}", - "interval": "", - "legendFormat": "Max:{{instance}}", - "range": true, - "refId": "B" - } - ], - "title": "JVM memory used", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 3, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "# for strimzi operator: remove kafka container java metrics\nsum without(gc)(rate(jvm_gc_collection_seconds_sum{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\", container!=\"kafka\" }[$__rate_interval]))", - "interval": "", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Time spent in GC", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 18, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "refId": "A" - } - ], - "title": "Request latency", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Amount of time it takes for the server to respond to a client request", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 18 - }, - "id": 9, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "zookeeper_minrequestlatency{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"} * ignoring (minrequestlatency,ticktime)zookeeper_ticktime{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{server_name}}:{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Request latency - min", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Amount of time it takes for the server to respond to a client request", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "decimals": 0, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 12000 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 18 - }, - "id": 10, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "zookeeper_avgrequestlatency{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"} * ignoring (avgrequestlatency,ticktime) zookeeper_ticktime{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{server_name}}:{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Request latency - average", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Amount of time it takes for the server to respond to a client request", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ ], - "mappings": [ ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 18 - }, - "id": 11, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.4.3", - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "zookeeper_maxrequestlatency{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"} * ignoring (maxrequestlatency,ticktime)\n zookeeper_ticktime{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{server_name}}:{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Request latency - max", - "type": "timeseries" - } - ], - "refresh": "30s", - "revision": 1, - "schemaVersion": 38, - "style": "dark", - "tags": [ - "kafka" - ], - "templating": { - "list": [ - { - "label": "Data source", - "name": "datasource", - "query": "prometheus", - "type": "datasource" - }, - { - "allValue": ".+", - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "query": "label_values(zookeeper_outstandingrequests{job=~\"integrations/kafka-zookeeper|integrations/kafka\"}, job)", - "refresh": 2, - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "includeAll": true, - "label": "Kafka_cluster", - "multi": true, - "name": "kafka_cluster", - "query": "label_values(zookeeper_outstandingrequests{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\"}, kafka_cluster)", - "refresh": 2, - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "includeAll": true, - "label": "Instance", - "multi": true, - "name": "instance", - "query": "label_values(zookeeper_outstandingrequests{job=~\"integrations/kafka-zookeeper|integrations/kafka\",job=~\"$job\",kafka_cluster=~\"$kafka_cluster\"}, instance)", - "refresh": 2, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Zookeeper overview", - "uid": "H4xS98vWk", - "version": 12, - "weekStart": "" -} diff --git a/assets/kafka/rules.yaml b/assets/kafka/rules.yaml index 19765bd..0967ef4 100644 --- a/assets/kafka/rules.yaml +++ b/assets/kafka/rules.yaml @@ -1 +1 @@ -null +{} diff --git a/assets/kubernetes/alerts.yaml b/assets/kubernetes/alerts.yaml index 3c316e8..94afe9b 100644 --- a/assets/kubernetes/alerts.yaml +++ b/assets/kubernetes/alerts.yaml @@ -115,7 +115,7 @@ groups: summary: StatefulSet update has not been rolled out. expr: | ( - max without (revision) ( + max by(namespace, statefulset, job, cluster) ( kube_statefulset_status_current_revision{job="kube-state-metrics"} unless kube_statefulset_status_update_revision{job="kube-state-metrics"} @@ -514,9 +514,9 @@ groups: runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn summary: The API server is burning too much error budget. expr: | - sum(apiserver_request:burnrate1h) > (14.40 * 0.01000) - and - sum(apiserver_request:burnrate5m) > (14.40 * 0.01000) + sum by(cluster) (apiserver_request:burnrate1h) > (14.40 * 0.01000) + and on(cluster) + sum by(cluster) (apiserver_request:burnrate5m) > (14.40 * 0.01000) for: 2m labels: long: 1h @@ -528,9 +528,9 @@ groups: runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn summary: The API server is burning too much error budget. expr: | - sum(apiserver_request:burnrate6h) > (6.00 * 0.01000) - and - sum(apiserver_request:burnrate30m) > (6.00 * 0.01000) + sum by(cluster) (apiserver_request:burnrate6h) > (6.00 * 0.01000) + and on(cluster) + sum by(cluster) (apiserver_request:burnrate30m) > (6.00 * 0.01000) for: 15m labels: long: 6h @@ -542,9 +542,9 @@ groups: runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn summary: The API server is burning too much error budget. expr: | - sum(apiserver_request:burnrate1d) > (3.00 * 0.01000) - and - sum(apiserver_request:burnrate2h) > (3.00 * 0.01000) + sum by(cluster) (apiserver_request:burnrate1d) > (3.00 * 0.01000) + and on(cluster) + sum by(cluster) (apiserver_request:burnrate2h) > (3.00 * 0.01000) for: 1h labels: long: 1d @@ -556,9 +556,9 @@ groups: runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn summary: The API server is burning too much error budget. expr: | - sum(apiserver_request:burnrate3d) > (1.00 * 0.01000) - and - sum(apiserver_request:burnrate6h) > (1.00 * 0.01000) + sum by(cluster) (apiserver_request:burnrate3d) > (1.00 * 0.01000) + and on(cluster) + sum by(cluster) (apiserver_request:burnrate6h) > (1.00 * 0.01000) for: 3h labels: long: 3d @@ -573,7 +573,7 @@ groups: runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration summary: Client certificate is about to expire. expr: | - apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 604800 + apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(cluster, job) histogram_quantile(0.01, sum by (cluster, job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 604800 for: 5m labels: severity: warning @@ -584,7 +584,7 @@ groups: runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration summary: Client certificate is about to expire. expr: | - apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 86400 + apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(cluster, job) histogram_quantile(0.01, sum by (cluster, job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 86400 for: 5m labels: severity: critical @@ -628,7 +628,7 @@ groups: summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests. expr: | - sum(rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="kube-apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m])) ) > 0.20 + sum by(cluster) (rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m])) / ( sum by(cluster) (rate(apiserver_request_total{job="kube-apiserver"}[10m])) + sum by(cluster) (rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m])) ) > 0.20 for: 5m labels: severity: warning diff --git a/assets/kubernetes/dashboards/apiserver.json b/assets/kubernetes/dashboards/apiserver.json index 84dcb1d..f78083f 100644 --- a/assets/kubernetes/dashboards/apiserver.json +++ b/assets/kubernetes/dashboards/apiserver.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -17,7 +30,7 @@ "options": { "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only." }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "title": "Notice", "type": "text" }, @@ -41,7 +54,7 @@ }, "id": 2, "interval": "1m", - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -87,7 +100,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -121,7 +134,7 @@ }, "id": 4, "interval": "1m", - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -219,7 +232,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -263,7 +276,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -306,7 +319,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -340,7 +353,7 @@ }, "id": 8, "interval": "1m", - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -438,7 +451,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -482,7 +495,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -525,7 +538,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -568,7 +581,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -611,7 +624,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -657,7 +670,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -699,7 +712,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -742,7 +755,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -784,7 +797,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -800,7 +813,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/cluster-total.json b/assets/kubernetes/dashboards/cluster-total.json index 52ea453..45ad32f 100644 --- a/assets/kubernetes/dashboards/cluster-total.json +++ b/assets/kubernetes/dashboards/cluster-total.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -33,14 +46,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (namespace) (\n rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -79,14 +92,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (namespace) (\n rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -150,14 +163,14 @@ "y": 9 }, "id": 3, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (namespace) (\n rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "format": "table", "instant": true }, @@ -166,7 +179,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (namespace) (\n rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "format": "table", "instant": true }, @@ -175,7 +188,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (namespace) (rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "avg by (namespace) (\n rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "format": "table", "instant": true }, @@ -184,7 +197,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (namespace) (rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "avg by (namespace) (\n rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "format": "table", "instant": true }, @@ -193,7 +206,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_receive_packets_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (namespace) (\n rate(container_network_receive_packets_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "format": "table", "instant": true }, @@ -202,7 +215,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (namespace) (\n rate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "format": "table", "instant": true }, @@ -211,7 +224,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (namespace) (\n rate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "format": "table", "instant": true }, @@ -220,7 +233,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (namespace) (\n rate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "format": "table", "instant": true } @@ -315,14 +328,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (namespace) (rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "avg by (namespace) (\n rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -361,14 +374,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (namespace) (rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "avg by (namespace) (\n rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -407,14 +420,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (namespace) (\n rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -453,14 +466,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (namespace) (\n rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -499,14 +512,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_receive_packets_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (namespace) (\n rate(container_network_receive_packets_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -545,14 +558,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (namespace) (\n rate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -591,14 +604,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (namespace) (\n rate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -637,14 +650,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (namespace) (\n rate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -683,14 +696,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (instance) (rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$__rate_interval]) / rate(node_netstat_Tcp_OutSegs{cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "sum by (instance) (\n rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$__rate_interval]) / rate(node_netstat_Tcp_OutSegs{cluster=\"$cluster\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -729,14 +742,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (instance) (rate(node_netstat_TcpExt_TCPSynRetrans{cluster=\"$cluster\"}[$__rate_interval]) / rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "sum by (instance) (\n rate(node_netstat_TcpExt_TCPSynRetrans{cluster=\"$cluster\"}[$__rate_interval]) / rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -745,7 +758,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/controller-manager.json b/assets/kubernetes/dashboards/controller-manager.json index 88c2e15..eaa4fc8 100644 --- a/assets/kubernetes/dashboards/controller-manager.json +++ b/assets/kubernetes/dashboards/controller-manager.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -22,7 +35,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -73,7 +86,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -124,7 +137,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -175,7 +188,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -226,7 +239,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -301,7 +314,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -352,7 +365,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -403,7 +416,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -454,7 +467,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -505,7 +518,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -521,7 +534,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/k8s-resources-cluster.json b/assets/kubernetes/dashboards/k8s-resources-cluster.json index e8d7293..5891ed7 100644 --- a/assets/kubernetes/dashboards/k8s-resources-cluster.json +++ b/assets/kubernetes/dashboards/k8s-resources-cluster.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -8,7 +21,7 @@ }, "fieldConfig": { "defaults": { - "unit": "none" + "unit": "percentunit" } }, "gridPos": { @@ -22,7 +35,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -57,7 +70,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -92,7 +105,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -127,7 +140,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -162,7 +175,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -197,7 +210,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -247,7 +260,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -306,7 +319,7 @@ "y": 12 }, "id": 8, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -463,7 +476,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -558,7 +571,7 @@ "y": 24 }, "id": 10, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -735,7 +748,7 @@ "y": 30 }, "id": 11, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -879,7 +892,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -930,7 +943,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -981,7 +994,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1032,7 +1045,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1083,7 +1096,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1134,7 +1147,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1185,7 +1198,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1236,7 +1249,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1287,7 +1300,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1338,7 +1351,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1409,7 +1422,7 @@ "y": 96 }, "id": 22, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1518,7 +1531,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/k8s-resources-namespace.json b/assets/kubernetes/dashboards/k8s-resources-namespace.json index 4ebf68d..c2a6a4f 100644 --- a/assets/kubernetes/dashboards/k8s-resources-namespace.json +++ b/assets/kubernetes/dashboards/k8s-resources-namespace.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -22,7 +35,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -57,7 +70,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -92,7 +105,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -127,7 +140,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -229,7 +242,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -244,7 +257,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "expr": "scalar(max(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"}))", "legendFormat": "quota - requests" }, { @@ -252,7 +265,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "expr": "scalar(max(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"}))", "legendFormat": "quota - limits" } ], @@ -304,7 +317,7 @@ "y": 14 }, "id": 6, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -487,7 +500,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -502,7 +515,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "expr": "scalar(max(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"}))", "legendFormat": "quota - requests" }, { @@ -510,7 +523,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "expr": "scalar(max(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"}))", "legendFormat": "quota - limits" } ], @@ -565,7 +578,7 @@ "y": 28 }, "id": 8, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -755,7 +768,7 @@ "y": 35 }, "id": 9, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -899,7 +912,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -950,7 +963,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1001,7 +1014,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1052,7 +1065,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1103,7 +1116,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1154,7 +1167,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1205,7 +1218,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1256,7 +1269,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1327,7 +1340,7 @@ "y": 70 }, "id": 18, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1436,7 +1449,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/k8s-resources-node.json b/assets/kubernetes/dashboards/k8s-resources-node.json index e4669c2..9fbdf51 100644 --- a/assets/kubernetes/dashboards/k8s-resources-node.json +++ b/assets/kubernetes/dashboards/k8s-resources-node.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -81,14 +94,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})", + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", job=\"kube-state-metrics\", node=~\"$node\", resource=\"cpu\"})", "legendFormat": "max capacity" }, { @@ -148,7 +161,7 @@ "y": 6 }, "id": 2, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -310,14 +323,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})", + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", job=\"kube-state-metrics\", node=~\"$node\", resource=\"memory\"})", "legendFormat": "max capacity" }, { @@ -380,7 +393,7 @@ "y": 18 }, "id": 4, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -496,7 +509,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/k8s-resources-pod.json b/assets/kubernetes/dashboards/k8s-resources-pod.json index ddf25a2..030e1b3 100644 --- a/assets/kubernetes/dashboards/k8s-resources-pod.json +++ b/assets/kubernetes/dashboards/k8s-resources-pod.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -89,7 +102,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -195,7 +208,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -237,7 +250,7 @@ "y": 14 }, "id": 3, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -420,7 +433,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -481,7 +494,7 @@ "y": 28 }, "id": 5, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -651,7 +664,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -702,7 +715,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -753,7 +766,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -804,7 +817,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -855,7 +868,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -906,7 +919,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -957,7 +970,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1016,7 +1029,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1075,7 +1088,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1126,7 +1139,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1180,7 +1193,7 @@ "y": 70 }, "id": 16, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1289,7 +1302,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/k8s-resources-workload.json b/assets/kubernetes/dashboards/k8s-resources-workload.json index d692fa2..03f4de8 100644 --- a/assets/kubernetes/dashboards/k8s-resources-workload.json +++ b/assets/kubernetes/dashboards/k8s-resources-workload.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -37,7 +50,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -96,7 +109,7 @@ "y": 7 }, "id": 2, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -227,7 +240,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -289,7 +302,7 @@ "y": 21 }, "id": 4, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -440,7 +453,7 @@ "y": 28 }, "id": 5, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -584,7 +597,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -635,7 +648,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -686,7 +699,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -737,7 +750,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -788,7 +801,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -839,7 +852,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -890,7 +903,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -941,7 +954,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -957,7 +970,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/k8s-resources-workloads-namespace.json b/assets/kubernetes/dashboards/k8s-resources-workloads-namespace.json index 2e4a1e5..39bb9b7 100644 --- a/assets/kubernetes/dashboards/k8s-resources-workloads-namespace.json +++ b/assets/kubernetes/dashboards/k8s-resources-workloads-namespace.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -89,7 +102,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -104,7 +117,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "expr": "scalar(max(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=~\"requests.cpu|cpu\"}))", "legendFormat": "quota - requests" }, { @@ -112,7 +125,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "expr": "scalar(max(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=~\"limits.cpu\"}))", "legendFormat": "quota - limits" } ], @@ -176,7 +189,7 @@ "y": 7 }, "id": 2, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -384,7 +397,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -399,7 +412,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "expr": "scalar(max(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=~\"requests.memory|memory\"}))", "legendFormat": "quota - requests" }, { @@ -407,7 +420,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "expr": "scalar(max(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=~\"limits.memory\"}))", "legendFormat": "quota - limits" } ], @@ -474,7 +487,7 @@ "y": 21 }, "id": 4, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -650,7 +663,7 @@ "y": 28 }, "id": 5, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -794,7 +807,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -845,7 +858,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -896,7 +909,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -947,7 +960,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -998,7 +1011,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1049,7 +1062,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1100,7 +1113,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1151,7 +1164,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1167,7 +1180,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/kubelet.json b/assets/kubernetes/dashboards/kubelet.json index 5eefede..def79f0 100644 --- a/assets/kubernetes/dashboards/kubelet.json +++ b/assets/kubernetes/dashboards/kubelet.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -22,7 +35,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -57,7 +70,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -92,7 +105,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -127,7 +140,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -162,7 +175,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -197,7 +210,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -248,7 +261,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -299,7 +312,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -350,7 +363,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -401,7 +414,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -460,7 +473,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -519,7 +532,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -570,7 +583,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -621,7 +634,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -672,7 +685,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -723,7 +736,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -774,7 +787,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -825,7 +838,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -876,7 +889,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -927,7 +940,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1002,7 +1015,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1053,7 +1066,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1104,7 +1117,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1155,7 +1168,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -1171,7 +1184,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/namespace-by-pod.json b/assets/kubernetes/dashboards/namespace-by-pod.json index 02b6c45..9c76406 100644 --- a/assets/kubernetes/dashboards/namespace-by-pod.json +++ b/assets/kubernetes/dashboards/namespace-by-pod.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -40,14 +53,14 @@ "y": 0 }, "id": 1, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "sum (\n rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -93,14 +106,14 @@ "y": 0 }, "id": 2, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "sum (\n rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -164,14 +177,14 @@ "y": 9 }, "id": 3, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum by (pod) (\n rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "format": "table", "instant": true }, @@ -180,7 +193,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum by (pod) (\n rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "format": "table", "instant": true }, @@ -189,7 +202,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum by (pod) (\n rate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "format": "table", "instant": true }, @@ -198,7 +211,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum by (pod) (\n rate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "format": "table", "instant": true }, @@ -207,7 +220,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum by (pod) (\n rate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "format": "table", "instant": true }, @@ -216,7 +229,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum by (pod) (\n rate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "format": "table", "instant": true } @@ -303,14 +316,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum by (pod) (\n rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -349,14 +362,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum by (pod) (\n rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -395,14 +408,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum by (pod) (\n rate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -441,14 +454,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum by (pod) (\n rate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -487,14 +500,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (namespace) (rate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval]))", + "expr": "sum by (pod) (\n rate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace!=\"\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -533,14 +546,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum by (pod) (\n rate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n * on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n)\n", "legendFormat": "__auto" } ], @@ -549,7 +562,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/namespace-by-workload.json b/assets/kubernetes/dashboards/namespace-by-workload.json index bccff6d..931786a 100644 --- a/assets/kubernetes/dashboards/namespace-by-workload.json +++ b/assets/kubernetes/dashboards/namespace-by-workload.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -26,14 +39,14 @@ "displayMode": "basic", "showUnfilled": false }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n* on (cluster,namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", "legendFormat": "__auto" } ], @@ -65,14 +78,14 @@ "displayMode": "basic", "showUnfilled": false }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n* on (cluster,namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", "legendFormat": "__auto" } ], @@ -136,14 +149,14 @@ "y": 9 }, "id": 3, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", + "expr": "sort_desc(sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod) kube_pod_info{cluster=\"$cluster\",namespace=\"$namespace\",host_network=\"false\"}\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", "format": "table", "instant": true }, @@ -152,7 +165,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", + "expr": "sort_desc(sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod) kube_pod_info{cluster=\"$cluster\",namespace=\"$namespace\",host_network=\"false\"}\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", "format": "table", "instant": true }, @@ -161,7 +174,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(avg(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", + "expr": "sort_desc(avg(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod) kube_pod_info{cluster=\"$cluster\",namespace=\"$namespace\",host_network=\"false\"}\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", "format": "table", "instant": true }, @@ -170,7 +183,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(avg(rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", + "expr": "sort_desc(avg(rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod) kube_pod_info{cluster=\"$cluster\",namespace=\"$namespace\",host_network=\"false\"}\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", "format": "table", "instant": true }, @@ -179,7 +192,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(sum(rate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", + "expr": "sort_desc(sum(rate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod) kube_pod_info{cluster=\"$cluster\",namespace=\"$namespace\",host_network=\"false\"}\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", "format": "table", "instant": true }, @@ -188,7 +201,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(sum(rate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", + "expr": "sort_desc(sum(rate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod) kube_pod_info{cluster=\"$cluster\",namespace=\"$namespace\",host_network=\"false\"}\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", "format": "table", "instant": true }, @@ -197,7 +210,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(sum(rate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", + "expr": "sort_desc(sum(rate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod) kube_pod_info{cluster=\"$cluster\",namespace=\"$namespace\",host_network=\"false\"}\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", "format": "table", "instant": true }, @@ -206,7 +219,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(sum(rate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", + "expr": "sort_desc(sum(rate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod) kube_pod_info{cluster=\"$cluster\",namespace=\"$namespace\",host_network=\"false\"}\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload, workload_type))\n", "format": "table", "instant": true } @@ -322,14 +335,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n* on (cluster,namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", "legendFormat": "__auto" } ], @@ -373,14 +386,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n* on (cluster,namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", "legendFormat": "__auto" } ], @@ -424,14 +437,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(avg(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", + "expr": "sort_desc(avg(rate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n* on (cluster,namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", "legendFormat": "__auto" } ], @@ -475,14 +488,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(avg(rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", + "expr": "sort_desc(avg(rate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n* on (cluster,namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", "legendFormat": "__auto" } ], @@ -526,14 +539,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(sum(rate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(rate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n* on (cluster,namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", "legendFormat": "__auto" } ], @@ -577,14 +590,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(sum(rate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(rate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n* on (cluster,namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", "legendFormat": "__auto" } ], @@ -628,14 +641,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(sum(rate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(rate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n* on (cluster,namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", "legendFormat": "__auto" } ], @@ -679,14 +692,14 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sort_desc(sum(rate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(rate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])\n* on (cluster,namespace,pod) group_left ()\n topk by (cluster,namespace,pod) (\n 1,\n max by (cluster,namespace,pod) (kube_pod_info{host_network=\"false\"})\n )\n* on (cluster,namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n", "legendFormat": "__auto" } ], @@ -695,7 +708,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/persistentvolumesusage.json b/assets/kubernetes/dashboards/persistentvolumesusage.json index 40ef01e..db3d4c8 100644 --- a/assets/kubernetes/dashboards/persistentvolumesusage.json +++ b/assets/kubernetes/dashboards/persistentvolumesusage.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -37,7 +50,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -99,7 +112,7 @@ }, "id": 2, "interval": "1m", - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -149,7 +162,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -211,7 +224,7 @@ }, "id": 4, "interval": "1m", - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -227,7 +240,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/pod-total.json b/assets/kubernetes/dashboards/pod-total.json index fb02fa8..55dbce6 100644 --- a/assets/kubernetes/dashboards/pod-total.json +++ b/assets/kubernetes/dashboards/pod-total.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -40,7 +53,7 @@ "y": 0 }, "id": 1, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -93,7 +106,7 @@ "y": 0 }, "id": 2, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -139,7 +152,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -185,7 +198,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -231,7 +244,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -277,7 +290,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -323,7 +336,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -369,7 +382,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -385,7 +398,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/proxy.json b/assets/kubernetes/dashboards/proxy.json index 3405f73..145340d 100644 --- a/assets/kubernetes/dashboards/proxy.json +++ b/assets/kubernetes/dashboards/proxy.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -22,7 +35,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -73,7 +86,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -124,7 +137,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -175,7 +188,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -226,7 +239,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -277,7 +290,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -352,7 +365,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -403,7 +416,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -454,7 +467,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -505,7 +518,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -556,7 +569,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -572,7 +585,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/scheduler.json b/assets/kubernetes/dashboards/scheduler.json index 08dc644..a8a57fa 100644 --- a/assets/kubernetes/dashboards/scheduler.json +++ b/assets/kubernetes/dashboards/scheduler.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -22,7 +35,7 @@ "options": { "colorMode": "none" }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -73,7 +86,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -148,7 +161,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -223,7 +236,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -298,7 +311,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -349,7 +362,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -400,7 +413,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -451,7 +464,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -502,7 +515,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -518,7 +531,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/dashboards/workload-total.json b/assets/kubernetes/dashboards/workload-total.json index 8087958..b0be78f 100644 --- a/assets/kubernetes/dashboards/workload-total.json +++ b/assets/kubernetes/dashboards/workload-total.json @@ -1,5 +1,18 @@ { "editable": false, + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "kubernetes-mixin" + ], + "targetBlank": false, + "title": "Kubernetes", + "type": "dashboards" + } + ], "panels": [ { "datasource": { @@ -26,7 +39,7 @@ "displayMode": "basic", "showUnfilled": false }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -65,7 +78,7 @@ "displayMode": "basic", "showUnfilled": false }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -104,7 +117,7 @@ "displayMode": "basic", "showUnfilled": false }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -143,7 +156,7 @@ "displayMode": "basic", "showUnfilled": false }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -194,7 +207,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -245,7 +258,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -296,7 +309,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -347,7 +360,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -398,7 +411,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -449,7 +462,7 @@ "mode": "single" } }, - "pluginVersion": "v10.4.0", + "pluginVersion": "v11.1.0", "targets": [ { "datasource": { @@ -465,7 +478,7 @@ } ], "refresh": "10s", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "kubernetes-mixin" ], diff --git a/assets/kubernetes/rules.yaml b/assets/kubernetes/rules.yaml index 2af056a..b1bccaa 100644 --- a/assets/kubernetes/rules.yaml +++ b/assets/kubernetes/rules.yaml @@ -15,18 +15,18 @@ groups: labels: verb: write record: code:apiserver_request_total:increase30d - - expr: | - sum by (cluster, verb, scope) (increase(apiserver_request_sli_duration_seconds_count{job="kube-apiserver"}[1h])) - record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h - - expr: | - sum by (cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h[30d]) * 24 * 30) - record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d - expr: | sum by (cluster, verb, scope, le) (increase(apiserver_request_sli_duration_seconds_bucket[1h])) record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h - expr: | sum by (cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h[30d]) * 24 * 30) record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d + - expr: | + sum by (cluster, verb, scope) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h{le="+Inf"}) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h + - expr: | + sum by (cluster, verb, scope) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{le="+Inf"} * 24 * 30) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d - expr: | 1 - ( ( @@ -481,7 +481,7 @@ groups: max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) ) record: node_namespace_pod_container:container_memory_swap -- name: k8s.rules.container_resource +- name: k8s.rules.container_memory_requests rules: - expr: | kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) @@ -500,6 +500,8 @@ groups: ) ) record: namespace_memory:kube_pod_container_resource_requests:sum +- name: k8s.rules.container_cpu_requests + rules: - expr: | kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( @@ -517,6 +519,8 @@ groups: ) ) record: namespace_cpu:kube_pod_container_resource_requests:sum +- name: k8s.rules.container_memory_limits + rules: - expr: | kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( @@ -534,6 +538,8 @@ groups: ) ) record: namespace_memory:kube_pod_container_resource_limits:sum +- name: k8s.rules.container_cpu_limits + rules: - expr: | kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( diff --git a/assets/loki/alerts.yaml b/assets/loki/alerts.yaml index 77f285b..c6a45ce 100644 --- a/assets/loki/alerts.yaml +++ b/assets/loki/alerts.yaml @@ -3,28 +3,31 @@ groups: rules: - alert: LokiRequestErrors annotations: - message: | - {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors. + description: | + {{ $labels.cluster }} {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors. + summary: Loki request error rate is high. expr: | - 100 * sum(rate(loki_request_duration_seconds_count{status_code=~"5.."}[2m])) by (namespace, job, route) + 100 * sum(rate(loki_request_duration_seconds_count{status_code=~"5.."}[2m])) by (cluster, namespace, job, route) / - sum(rate(loki_request_duration_seconds_count[2m])) by (namespace, job, route) + sum(rate(loki_request_duration_seconds_count[2m])) by (cluster, namespace, job, route) > 10 for: 15m labels: severity: critical - alert: LokiRequestPanics annotations: - message: | - {{ $labels.job }} is experiencing {{ printf "%.2f" $value }}% increase of panics. + description: | + {{ $labels.cluster }} {{ $labels.job }} is experiencing {{ printf "%.2f" $value }}% increase of panics. + summary: Loki requests are causing code panics. expr: | - sum(increase(loki_panic_total[10m])) by (namespace, job) > 0 + sum(increase(loki_panic_total[10m])) by (cluster, namespace, job) > 0 labels: severity: critical - alert: LokiRequestLatency annotations: - message: | - {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency. + description: | + {{ $labels.cluster }} {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency. + summary: Loki request error latency is high. expr: | cluster_namespace_job_route:loki_request_duration_seconds:99quantile{route!~"(?i).*tail.*|/schedulerpb.SchedulerForQuerier/QuerierLoop"} > 1 for: 15m @@ -32,10 +35,44 @@ groups: severity: critical - alert: LokiTooManyCompactorsRunning annotations: - message: | + description: | {{ $labels.cluster }} {{ $labels.namespace }} has had {{ printf "%.0f" $value }} compactors running for more than 5m. Only one compactor should run at a time. + summary: Loki deployment is running more than one compactor. expr: | - sum(loki_boltdb_shipper_compactor_running) by (namespace, cluster) > 1 + sum(loki_boltdb_shipper_compactor_running) by (cluster, namespace) > 1 for: 5m labels: severity: warning + - alert: LokiCompactorHasNotSuccessfullyRunCompaction + annotations: + description: | + {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3 hours since the last compaction. This may indicate a problem with the compactor. + summary: Loki compaction has not run in the last 3 hours since the last compaction. + expr: | + # The "last successful run" metric is updated even if the compactor owns no tenants, + # so this alert correctly doesn't fire if compactor has nothing to do. + min ( + time() - (loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{} > 0) + ) + by (cluster, namespace) + > 60 * 60 * 3 + for: 1h + labels: + severity: critical + - alert: LokiCompactorHasNotSuccessfullyRunCompaction + annotations: + description: | + {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3h since startup. This may indicate a problem with the compactor. + summary: Loki compaction has not run in the last 3h since startup. + expr: | + # The "last successful run" metric is updated even if the compactor owns no tenants, + # so this alert correctly doesn't fire if compactor has nothing to do. + max( + max_over_time( + loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{}[3h] + ) + ) by (cluster, namespace) + == 0 + for: 1h + labels: + severity: critical diff --git a/assets/loki/dashboards/loki-bloom-build.json b/assets/loki/dashboards/loki-bloom-build.json new file mode 100644 index 0000000..48d1884 --- /dev/null +++ b/assets/loki/dashboards/loki-bloom-build.json @@ -0,0 +1,6416 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "loki" + ], + "targetBlank": false, + "title": "Loki Dashboards", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 111, + "panels": [ ], + "targets": [ ], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Cell-wide compaction progress. Should increase till completion throughout each compaction period.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(Planned|success|failure)/" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_tenant_tasks_completed{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"})\n/\nsum(loki_bloomplanner_tenant_tasks_planned{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"})", + "hide": false, + "instant": false, + "legendFormat": "Progress", + "range": true, + "refId": "D" + } + ], + "title": "Overall progress", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Cell-wide compaction progress. Should increase till completion throughout each compaction period.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(Planned|success|failure)/" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 116, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (tenant) (loki_bloomplanner_tenant_tasks_completed{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"})\n/\nsum by (tenant) (loki_bloomplanner_tenant_tasks_planned{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"})", + "hide": false, + "instant": false, + "legendFormat": "{{tenant}}", + "range": true, + "refId": "D" + } + ], + "title": "Progress by tenant", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "description": "Blooms size vs uncompressed chunk size.", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 17, + "x": 0, + "y": 8 + }, + "id": 51, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "editorMode": "code", + "expr": "{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} |= \"level=error\" |= \"component=bloom-planner\"", + "queryType": "range", + "refId": "B" + } + ], + "title": "Errors Planner", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 17, + "y": 8 + }, + "id": 53, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "editorMode": "code", + "expr": "sum(count_over_time({cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} |= \"level=error\" |= \"component=bloom-planner\" [$__auto]))", + "legendFormat": "Error rate", + "queryType": "range", + "refId": "A" + } + ], + "title": "Errors Rate Planner", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "description": "Blooms size vs uncompressed chunk size.", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 17, + "x": 0, + "y": 15 + }, + "id": 133, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "editorMode": "code", + "expr": "{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} |= \"level=error\" |= \"component=bloom-builder\"", + "queryType": "range", + "refId": "B" + } + ], + "title": "Errors Builder", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 17, + "y": 15 + }, + "id": 134, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "editorMode": "code", + "expr": "sum(count_over_time({cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} |= \"level=error\" |= \"component=bloom-builder\" [$__auto]))", + "legendFormat": "Error rate", + "queryType": "range", + "refId": "A" + } + ], + "title": "Errors Rate Builder", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 112, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(success|failure)/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 100 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Planned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 15 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - failure" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Queued" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 125, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_tenant_tasks_planned{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Planned", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status) (loki_bloomplanner_tenant_tasks_completed{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Completed - {{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "loki_bloomplanner_inflight_tasks{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", quantile=\"0.95\"}", + "hide": false, + "instant": false, + "legendFormat": "inflight p95", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_queue_length{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Queued", + "range": true, + "refId": "D" + } + ], + "title": "Tasks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(success|failure)/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 100 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Planned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 15 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - failure" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Connected builders" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "IDLE Builders" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Builders processing task" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 126, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_connected_builders{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "hide": false, + "instant": false, + "legendFormat": "Connected builders", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(loki_bloombuilder_processing_task{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Builders processing task", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(loki_bloombuilder_processing_task{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} == 0)", + "hide": false, + "instant": false, + "legendFormat": "IDLE Builders", + "range": true, + "refId": "D" + } + ], + "title": "Tasks per builder", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 81, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.75, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p75", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" + } + ], + "title": "Series per task (includes series copied from other blocks)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 91, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" + } + ], + "title": "Number of bytes from chunks added to blocks during each compaction.", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 117, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Identify the tenant using the **_Progress by tenant_** panel from the overview and set tenant variable", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ ], + "title": "Tip", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(success|failure)/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 100 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Planned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 15 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - failure" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Queued" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 81 + }, + "id": 114, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_tenant_tasks_planned{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", tenant=\"$tenant\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Planned", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status) (loki_bloomplanner_tenant_tasks_completed{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", tenant=\"$tenant\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Completed - {{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_queue_length{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", user=\"$tenant\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Queued", + "range": true, + "refId": "C" + } + ], + "title": "Tasks per tenant", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 81 + }, + "id": 115, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "editorMode": "code", + "expr": "{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"}\n|= \"level=error\"\n|= \"tenant=$tenant\"", + "queryType": "range", + "refId": "B" + } + ], + "title": "Tenant errors", + "type": "logs" + } + ], + "targets": [ ], + "title": "Tasks", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 95, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How many tokens each builder is appending to blooms. Accounts for tokens that are not actually added to the blooms since they are already there. See the panel on the right for a drill down on the collision.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 90 + }, + "id": 96, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_tokens_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))\n/\nsum(count(loki_bloom_tokens_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}))", + "hide": false, + "instant": false, + "legendFormat": "Per core", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "C" + } + ], + "title": "Tokens rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Collision type may be `false` (no collision), `cache` (found in token cache) or true (found in bloom filter).\n\nType may be either `raw` (the original ngram) or `chunk_prefixed` (the ngram with the chunk prefix)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 90 + }, + "id": 97, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# tokens/s by type+collision\nsum by (collision) (\n rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n) \n/ on () group_left\nsum (\n rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "tokens/s by collision type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The sizes of the blooms created by the compactor. We build one bloom per series. The more unique ngrams and chunks the series has, the bigger their blooms will be.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 98 + }, + "id": 98, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(\n 1.0,\n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "max", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(\n 0.50, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "F" + } + ], + "title": "Bloom size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How many chunks are we indexing in the blooms. Either:\n- `copied` from a pre-existing bloom block, or \n- `iterated` through all its entries if processed for the first time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 98 + }, + "id": 99, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# chunks indexed, by iteration or copied from a pre-existing bloom\nsum(rate(loki_bloom_chunks_indexed_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])) by (type)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Chunks indexed", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Bloom building", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 56, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Shows the expected number of cpu cores we need to provision to build blooms as fast as we ingest data so a build iteration doesn't take longer than the build interval.\n\nWe may decide to have more to speed up building blooms.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 2030 + }, + "id": 94, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# This query shows the expected number of cpu cores we need to not fall behind\n# building blooms for data we're ingesting.\n# conceptually, the formula is:\n# (cell_bytes * space_amplification / bloom_bytes_processed_per_core)\n\n# number of replicas needed\nsum(avg_over_time(loki_cell:bytes:rate1m{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n*\n## Space amplification (how much data do we write compared to what we ingest?)\n(\n # rep factor\n 3 *\n sum(\n # 1 - dedupe_ratio\n 1 - \n sum(rate(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n /\n sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n )\n)\n/\n(\nsum(rate(loki_bloombuilder_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))\n)", + "hide": false, + "instant": false, + "legendFormat": "Needed", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Available", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)\n*\ncount(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Provisioned", + "range": true, + "refId": "C" + } + ], + "title": "Required CPUs to not lag behind", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2030 + }, + "id": 72, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# MB/s/core chunk data processed\nsum(rate(loki_bloombuilder_chunk_series_size_sum{cluster=~\"$cluster\", job=~\"$namespace/bloom-builder\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "B" + } + ], + "title": "MB/s per core", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 2037 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile(\n 0.99,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval])\n)", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile(\n 0.50,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg(\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "Max", + "range": true, + "refId": "F" + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2037 + }, + "id": 75, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + } + ], + "title": "CPU per pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 2044 + }, + "id": 76, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"memory\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile (\n 0.99,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile (\n 0.50,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg (\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max (\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, + "instant": false, + "legendFormat": "Max", + "range": true, + "refId": "F" + } + ], + "title": "Memory (workingset)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2044 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"})", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"memory\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + } + ], + "title": "Memory per pod (workingset)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 2051 + }, + "id": 27, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[10m]\n )\n) > 0", + "instant": false, + "legendFormat": "Restarts", + "range": true, + "refId": "A" + } + ], + "title": "Container restarts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2051 + }, + "id": 77, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}\n )\n) > 0", + "instant": false, + "legendFormat": "{{reason}} / {{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Container restarts reason per pod", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Builder Resource Usage", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 118, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 2302 + }, + "id": 119, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile(\n 0.99,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}[$__rate_interval])\n)", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile(\n 0.50,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg(\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "E" + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2302 + }, + "id": 120, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + } + ], + "title": "CPU per pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 2309 + }, + "id": 121, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"memory\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile (\n 0.99,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}\n)", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile (\n 0.50,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg (\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "E" + } + ], + "title": "Memory (workingset)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2309 + }, + "id": 122, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"})", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"memory\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + } + ], + "title": "Memory per pod (workingset)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 2316 + }, + "id": 123, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}[10m]\n )\n) > 0", + "instant": false, + "legendFormat": "Restarts", + "range": true, + "refId": "A" + } + ], + "title": "Container restarts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2316 + }, + "id": 124, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}\n )\n) > 0", + "instant": false, + "legendFormat": "{{reason}} / {{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Container restarts reason per pod", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Planner Resource Usage", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 110, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 0, + "y": 2497 + }, + "id": 108, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(loki_bloombuilder_metas_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Metas", + "range": true, + "refId": "A" + } + ], + "title": "Created Metas", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 9, + "y": 2497 + }, + "id": 140, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(loki_bloombuilder_metas_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__range]))", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Metas", + "range": false, + "refId": "A" + } + ], + "title": "Created Metas", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2497 + }, + "id": 105, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (phase) (increase(loki_bloomplanner_metas_deleted_total{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Deleted during {{phase}}", + "range": true, + "refId": "A" + } + ], + "title": "Deleted Metas", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Metas building", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 103, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 0, + "y": 2505 + }, + "id": 107, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(loki_bloombuilder_blocks_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Blocks", + "range": true, + "refId": "A" + } + ], + "title": "Created Blocks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 9, + "y": 2505 + }, + "id": 139, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(loki_bloombuilder_blocks_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__range]))", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": "Blocks", + "range": false, + "refId": "A" + } + ], + "title": "Created Blocks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2505 + }, + "id": 106, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (phase) (increase(loki_bloomplanner_blocks_deleted_total{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Deleted during {{phase}}", + "range": true, + "refId": "A" + } + ], + "title": "Deleted Blocks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of overlapping bloom blocks reused when creating new blocks\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 2512 + }, + "id": 109, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(loki_bloombuilder_blocks_reused_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Blocks", + "range": true, + "refId": "A" + } + ], + "title": "Blocks reused", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Blocks building", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 135, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Is the retention currently running?", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "fieldMinMax": false, + "mappings": [ + { + "options": { + "0": { + "color": "yellow", + "index": 0, + "text": "No" + }, + "1": { + "color": "green", + "index": 1, + "text": "Yes" + } + }, + "type": "value" + } + ], + "max": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 0, + "y": 2573 + }, + "id": 136, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (cluster, namespace) (loki_bloomplanner_retention_running{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Running now?", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Is the retention currently running?", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [ ], + "max": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bool_yes_no" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 3, + "y": 2573 + }, + "id": 137, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (cluster, namespace) (loki_bloomplanner_retention_running{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Retention running", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How much time applying retention took", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2573 + }, + "id": 138, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, \n sum by (status, le) (\n rate(loki_bloomplanner_retention_time_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval])\n )\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Retention time", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Retention", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 62, + "panels": [ + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 2581 + }, + "id": 71, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "During the planning phase, the planner downloads the metas and TSDBs to build the plan.\n\nOnce all blocks and metas are built, the builder flushes them to the object store.\n\nAfter each iteration, the planner deletes the metas and blocks marked for deletion in the tombstones.", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ ], + "title": "", + "transparent": true, + "type": "text" + }, + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 13, + "w": 2, + "x": 0, + "y": 2585 + }, + "id": 63, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "---\n#### GCS\n", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ ], + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 2, + "y": 2585 + }, + "id": 61, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (container, status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} {{status_code}}", + "range": true, + "refId": "B" + } + ], + "title": "QPS Planner", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 2585 + }, + "id": 64, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, + "refId": "F" + } + ], + "title": "Latency Planner", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 2, + "y": 2592 + }, + "id": 127, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (container, status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} {{status_code}}", + "range": true, + "refId": "B" + } + ], + "title": "QPS Builder", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 2592 + }, + "id": 128, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, + "refId": "F" + } + ], + "title": "Latency Planner", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 14, + "w": 2, + "x": 0, + "y": 2598 + }, + "id": 65, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "---\n#### S3\n", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ ], + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 2, + "y": 2599 + }, + "id": 67, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} {{status_code}}", + "range": true, + "refId": "B" + } + ], + "title": "QPS Planner", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 2599 + }, + "id": 69, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", job=\"\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, + "refId": "F" + } + ], + "title": "Latency Planner", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 2, + "y": 2606 + }, + "id": 129, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} {{status_code}}", + "range": true, + "refId": "B" + } + ], + "title": "QPS Builder", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 2606 + }, + "id": 130, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, + "refId": "F" + } + ], + "title": "Latency Builder", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 0, + "y": 2612 + }, + "id": 66, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "---\n#### Azure\nBlob Storage", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ ], + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 2, + "y": 2613 + }, + "id": 68, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} {{status_code}}", + "range": true, + "refId": "B" + } + ], + "title": "QPS Planner", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 2613 + }, + "id": 70, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, + "refId": "F" + } + ], + "title": "Latency Planner", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 2, + "y": 2620 + }, + "id": 131, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} {{status_code}}", + "range": true, + "refId": "B" + } + ], + "title": "QPS Builder", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 2620 + }, + "id": 132, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, + "refId": "F" + } + ], + "title": "Latency Builder", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Object Store", + "type": "row" + } + ], + "preload": false, + "refresh": "10s", + "rows": [ ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "loki" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(loki_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "hide": 0, + "label": null, + "name": "loki_datasource", + "options": [ ], + "query": "loki", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Tenant", + "multi": false, + "name": "tenant", + "options": [ ], + "query": "label_values(loki_bloomplanner_tenant_tasks_planned{cluster=\"$cluster\", namespace=\"$namespace\"}, tenant)", + "refresh": 0, + "regex": "", + "sort": 3, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Loki / Bloom Build", + "uid": "bloom-build", + "version": 0, + "weekStart": "" +} diff --git a/assets/loki/dashboards/loki-bloom-gateway.json b/assets/loki/dashboards/loki-bloom-gateway.json new file mode 100644 index 0000000..ad2919b --- /dev/null +++ b/assets/loki/dashboards/loki-bloom-gateway.json @@ -0,0 +1,6022 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "loki" + ], + "targetBlank": false, + "title": "Loki Dashboards", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 73, + "panels": [ ], + "targets": [ ], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Percentage of chunks that are filtered by using bloom filters", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "yellow", + "value": 0.75 + }, + { + "color": "green", + "value": 0.90000000000000002 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Chunks", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Series", + "range": true, + "refId": "B" + } + ], + "title": "Filter ratio - Bloom Gateway (server)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Percentage of chunks that are filtered by using bloom filters", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "yellow", + "value": 0.75 + }, + { + "color": "green", + "value": 0.90000000000000002 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 75, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(loki_bloom_gateway_filtered_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_requested_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))", + "instant": true, + "legendFormat": "Chunks", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(loki_bloom_gateway_filtered_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_requested_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))", + "hide": false, + "instant": true, + "legendFormat": "Series", + "range": false, + "refId": "B" + } + ], + "title": "Filter ratio", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Desired" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 72, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "kube_statefulset_status_replicas_ready{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}", + "hide": false, + "instant": false, + "legendFormat": "Ready", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "kube_statefulset_replicas{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}", + "hide": false, + "instant": false, + "legendFormat": "Desired", + "range": true, + "refId": "B" + } + ], + "title": "Readiness", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Percentage of chunks that are filtered by using bloom filters", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "yellow", + "value": 0.75 + }, + { + "color": "green", + "value": 0.90000000000000002 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 93, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_gateway_querier_chunks_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "Chunks", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(loki_bloom_gateway_querier_series_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Series", + "range": true, + "refId": "B" + } + ], + "title": "Filter ratio - Index Gateway (client)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Percentage of chunks that are filtered by using bloom filters", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "yellow", + "value": 0.75 + }, + { + "color": "green", + "value": 0.90000000000000002 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 7 + }, + "id": 94, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(loki_bloom_gateway_querier_chunks_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))", + "instant": true, + "legendFormat": "Chunks", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(loki_bloom_gateway_querier_series_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))", + "hide": false, + "instant": true, + "legendFormat": "Series", + "range": false, + "refId": "B" + } + ], + "title": "Filter ratio", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 37, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n max by (pod, reason) (kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n * on (pod) group_left\n sum by (pod) (increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))\n) > 0", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{pod}} ({{reason}})", + "range": true, + "refId": "C" + } + ], + "title": "Container restarts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Percentage of chunks that are filtered by using bloom filters", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "yellow", + "value": 0.75 + }, + { + "color": "green", + "value": 0.90000000000000002 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 99, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n sum(rate(loki_index_gateway_prefilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)\n -\n sum(rate(loki_index_gateway_postfilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)\n)\n/\nsum(rate(loki_index_gateway_prefilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)", + "instant": false, + "legendFormat": "chunks {{ route}}", + "range": true, + "refId": "A" + } + ], + "title": "Filter ratio - Index Gateway by route", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.10000000000000001 + }, + { + "color": "#EF843C", + "value": 0.25 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 100, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(loki_bloom_gateway_querier_series_skipped_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "series", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(loki_bloom_gateway_querier_chunks_skipped_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "chunks", + "range": true, + "refId": "B" + } + ], + "title": "Data skipped because they don't match any bocks", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 96, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 447 + }, + "id": 97, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=~\"(found|skipped|missed)\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Found/Skipped/Missing chunks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 447 + }, + "id": 98, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"found\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Filtered chunks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 454 + }, + "id": 107, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=~\"(found|skipped|missed)\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Found/Skipped/Missing series", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 454 + }, + "id": 108, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"found\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Filtered series", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Bloom Recorder", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 95, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 9, + "w": 15, + "x": 0, + "y": 920 + }, + "id": 48, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "editorMode": "code", + "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=error\" or \"panic:\" | logfmt", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "editorMode": "code", + "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=warn\" | logfmt", + "queryType": "range", + "refId": "B" + } + ], + "title": "Errors", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "symlog" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "panic" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 9, + "x": 15, + "y": 920 + }, + "id": 52, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "editorMode": "code", + "expr": "sum by (level) (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |~ \"level=(warn|error)\" | logfmt [$__auto]))", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "editorMode": "code", + "expr": "sum (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"panic:\" | logfmt [$__auto]))", + "queryType": "range", + "refId": "B" + } + ], + "title": "Errors Rate", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Logs", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 56, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 1764 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1764 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"memory\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + } + ], + "title": "Memory (workingset)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2140 + }, + "id": 81, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})", + "hide": true, + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"memory\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"}) by (pod)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "D" + } + ], + "title": "Memory (heap inuse)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 2147 + }, + "id": 87, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (pod) (rate(go_gc_cycles_total_gc_cycles_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "GC rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 2147 + }, + "id": 88, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (pod) (rate(go_gc_duration_seconds_sum{container=\"bloom-gateway\"}[$__rate_interval]))\n/\nsum by (pod) (rate(go_gc_duration_seconds_count{container=\"bloom-gateway\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "GC duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 2147 + }, + "id": 89, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "GC pauses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 2154 + }, + "id": 84, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(node_disk_read_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_reads_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{pod}}", + "range": true, + "refId": "D" + } + ], + "title": "Disk reads", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 2154 + }, + "id": 85, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(node_disk_written_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)", + "hide": false, + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "D" + } + ], + "title": "Disk writes", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 2161 + }, + "id": 102, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(sum by (instance) (rate(node_disk_read_bytes_total[$__rate_interval]))\n+ on(instance) group_right() \n(count by (instance) (container_fs_reads_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0))", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Reads", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(sum by(instance) (rate(node_disk_written_bytes_total[$__rate_interval]))\n+ on(instance) group_right() \n(count by(instance) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)) * -1", + "hide": false, + "instant": false, + "legendFormat": "Writes", + "range": true, + "refId": "A" + } + ], + "title": "Disk reads/writes", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Resource usage", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 2, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1175 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status_code) (\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\",job=\"$namespace/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "QPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1175 + }, + "id": 86, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "QPS per Pod", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1249 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}))", + "hide": false, + "instant": false, + "legendFormat": "{{ route }} 50th percentile", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}) by (route) ", + "hide": false, + "instant": false, + "legendFormat": "{{ route }} Average", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}))", + "hide": false, + "instant": false, + "legendFormat": "{{ route }} 99th percentile", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}))", + "hide": false, + "instant": false, + "legendFormat": "{{ route }} max", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1249 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99,\n sum(\n rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n ) by (pod, le)\n )\n", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Per Pod Latency (p99)", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "QPS and Latency", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 58, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 1176 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (user) (loki_bloom_gateway_queue_length{cluster=\"$cluster\", namespace=\"$namespace\"})", + "hide": false, + "instant": false, + "legendFormat": "{{user}}", + "range": true, + "refId": "B" + } + ], + "title": "Queue Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "For how long do pending tasks stay in the queue", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 1176 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum (loki_bloom_gateway_queue_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"})\n/\nsum (loki_bloom_gateway_queue_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"})", + "hide": false, + "instant": false, + "legendFormat": "avg", + "range": true, + "refId": "D" + } + ], + "title": "Queue Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Inflight requests tracks all tasks both queued and in progress", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1176 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (quantile) (loki_bloom_gateway_inflight_tasks{cluster=\"$cluster\", namespace=\"$namespace\", quantile=\"0.99\"})", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Inflight tasks", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Task Queue", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 68, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1177 + }, + "id": 69, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "instant": false, + "legendFormat": "{{status}}-p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "hide": false, + "instant": false, + "legendFormat": "{{status}}-p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "hide": false, + "instant": false, + "legendFormat": "{{status}}-p90", + "range": true, + "refId": "C" + } + ], + "title": "Processing time for tasks (per worker iteration)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1177 + }, + "id": 70, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "instant": false, + "legendFormat": "{{status}}-p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "hide": false, + "instant": false, + "legendFormat": "{{status}}-p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "hide": false, + "instant": false, + "legendFormat": "{{status}}-p90", + "range": true, + "refId": "C" + } + ], + "title": "Block query latency (single block)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 1177 + }, + "id": 71, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_gateway_tasks_dequeued_total{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "dequeued", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status) (rate(loki_bloom_gateway_tasks_processed_total{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "processed {{status}}", + "range": true, + "refId": "B" + } + ], + "title": "Tasks dequeued/processed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1214 + }, + "id": 105, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_gateway_process_duration_seconds_count{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (status)", + "instant": false, + "legendFormat": "{{status}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Iterations per second", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Processing", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 59, + "panels": [ + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1178 + }, + "id": 19, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ ], + "title": "We cache bloom blocks in memory to prevent the gateway from hitting the object store too often", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1179 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_embeddedcache_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"}) by (pod)", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}) by (pod)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Cache size (per pod)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "linearThreshold": 1000, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.fillOpacity", + "value": 25 + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Items" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 25 + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "unit", + "value": "" + }, + { + "id": "custom.axisSoftMin", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1179 + }, + "id": 83, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloom_blocks_cache_entries{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})", + "hide": false, + "instant": false, + "legendFormat": "Items", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_blocks_cache_added_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Added", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_blocks_cache_evicted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (reason)", + "hide": false, + "instant": false, + "legendFormat": "Evicted ({{reason}})", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "hide": true, + "instant": false, + "legendFormat": "Size", + "range": true, + "refId": "E" + } + ], + "title": "Cache rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "linearThreshold": 1000, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "hit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "miss" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1186 + }, + "id": 92, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status) (\n rate(loki_bloom_blocks_cache_fetched_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])\n)\n/ ignoring(status) group_left\nsum (\n rate(loki_bloom_blocks_cache_fetched_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Hit/Miss ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Size (.*)/" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1193 + }, + "id": 76, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_embeddedcache_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})\n/\nsum(loki_embeddedcache_entries{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})", + "instant": false, + "legendFormat": "Size", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n/\nsum(loki_bloom_blocks_cache_entries{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})", + "hide": false, + "instant": false, + "legendFormat": "Size", + "range": true, + "refId": "B" + } + ], + "title": "Average item size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.* (blocks|metas) size/" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1193 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_store_metas_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "metas fetch rate", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_store_blocks_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "blocks fetch rate", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_blocks_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "p90 blocks size", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_metas_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "p90 metas size", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1.0, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "G" + } + ], + "title": "Bloom Store", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Size (.*)/" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1200 + }, + "id": 101, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (job)(rate(loki_bloom_store_download_queue_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Size", + "range": true, + "refId": "B" + } + ], + "title": "Block download queue size", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Blocks Cache", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 60, + "panels": [ + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1013 + }, + "id": 61, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ ], + "title": "The gateway download bloom meta files and blocks from the object store.", + "transparent": true, + "type": "text" + }, + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 0, + "y": 1014 + }, + "id": 24, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "---\n#### GCS\n", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ ], + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 2, + "y": 1014 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", + "instant": false, + "legendFormat": "{{operation}} {{status_code}}", + "range": true, + "refId": "A" + } + ], + "title": "QPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 1014 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, + "refId": "C" + } + ], + "title": "Latency", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 0, + "y": 1021 + }, + "id": 62, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "---\n#### S3\n", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ ], + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 2, + "y": 1021 + }, + "id": 63, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", + "instant": false, + "legendFormat": "{{operation}} {{status_code}}", + "range": true, + "refId": "A" + } + ], + "title": "QPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 1021 + }, + "id": 64, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, + "refId": "C" + } + ], + "title": "Latency", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 0, + "y": 1028 + }, + "id": 65, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "---\n#### Azure\nBlob Storage\n\n", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ ], + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 2, + "y": 1028 + }, + "id": 66, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", + "instant": false, + "legendFormat": "{{operation}} {{status_code}}", + "range": true, + "refId": "A" + } + ], + "title": "QPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 1028 + }, + "id": 67, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, + "refId": "C" + } + ], + "title": "Latency", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Object Store", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 77, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1044 + }, + "id": 80, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1.0000000000000001e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(loki_bloom_gateway_dequeue_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval])", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Dequeue duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1044 + }, + "id": 106, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1.0000000000000001e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77765", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "increase(loki_bloom_gateway_tasks_dequeued_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval])", + "format": "heatmap", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Dequeue count", + "type": "heatmap" + } + ], + "targets": [ ], + "title": "Misc", + "type": "row" + } + ], + "preload": false, + "refresh": "10s", + "rows": [ ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "loki" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(loki_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "hide": 0, + "label": null, + "name": "loki_datasource", + "options": [ ], + "query": "loki", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Loki / Bloom Gateway", + "uid": "bloom-gateway", + "version": 0, + "weekStart": "" +} diff --git a/assets/loki/dashboards/loki-chunks.json b/assets/loki/dashboards/loki-chunks.json index b8c0688..2e4b14b 100644 --- a/assets/loki/dashboards/loki-chunks.json +++ b/assets/loki/dashboards/loki-chunks.json @@ -27,156 +27,98 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "series", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Series", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}) / sum(loki_ingester_memory_streams{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "chunks", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Chunks per series", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -191,81 +133,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(loki_ingester_chunk_utilization_sum{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Utilization", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "percentunit", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -283,78 +211,64 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 4, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Age", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -387,81 +301,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(loki_ingester_chunk_entries_sum{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Log Entries Per Chunk", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -479,80 +379,51 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[5m])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[5m]))", + "expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Index Entries", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Index Entries Per Chunk", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -567,80 +438,51 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "cortex_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}", + "expr": "loki_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"} or cortex_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Queue Length", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { "aliasColors": { @@ -649,82 +491,196 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Flush Rate", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -739,138 +695,99 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 9, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Chunks Flushed/Second", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 10, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{reason}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Chunk Flush Reason", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "short", @@ -1027,96 +944,63 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 13, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 12, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p99", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[1m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p90", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[1m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p50", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Chunk Size Quantiles", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1131,96 +1015,63 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 14, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 12, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[5m])) by (le))", + "expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p50", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[5m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p99", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[5m])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[5m]))", + "expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster=\"$cluster\", job=~\"$namespace/ingester.*\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "avg", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Chunk Duration hours (end-start)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1244,7 +1095,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/loki/dashboards/loki-deletion.json b/assets/loki/dashboards/loki-deletion.json index 5d8cfb6..44073b2 100644 --- a/assets/loki/dashboards/loki-deletion.json +++ b/assets/loki/dashboards/loki-deletion.json @@ -62,7 +62,6 @@ "expr": "sum(loki_compactor_pending_delete_requests_count{cluster=~\"$cluster\", namespace=~\"$namespace\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], @@ -138,7 +137,6 @@ "expr": "max(loki_compactor_oldest_pending_delete_request_age_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], @@ -191,232 +189,145 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on() vector(0)) - on () (loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on () vector(0))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "in progress", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "# of Delete Requests (received - processed) ", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 4, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(increase(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "received", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Delete Requests Received / Day", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(increase(loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "processed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Delete Requests Processed / Day", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -431,232 +342,145 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"}", + "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Compactor CPU usage", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} / 1024 / 1024 ", + "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} / 1024 / 1024 ", "format": "time_series", - "intervalFactor": 2, "legendFormat": " {{pod}} ", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Compactor memory usage (MiB)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 8, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Compaction run duration (seconds)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -671,156 +495,98 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 9, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(increase(loki_compactor_load_pending_requests_attempts_total{status=\"fail\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[1h]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "failures", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Failures in Loading Delete Requests / Hour", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 10, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\",job=~\"$namespace/compactor\"}[$__rate_interval])) by (user)", + "expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}[$__rate_interval])) by (user)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{user}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Lines Deleted / Sec", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -840,7 +606,7 @@ "span": 6, "targets": [ { - "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ", + "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ", "refId": "A" } ], @@ -853,7 +619,7 @@ "span": 6, "targets": [ { - "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"", + "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"", "refId": "A" } ], @@ -882,7 +648,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -935,6 +701,16 @@ "tagsQuery": "", "type": "query", "useTags": false + }, + { + "hide": 0, + "label": null, + "name": "loki_datasource", + "options": [ ], + "query": "loki", + "refresh": 1, + "regex": "", + "type": "datasource" } ] }, diff --git a/assets/loki/dashboards/loki-logs.json b/assets/loki/dashboards/loki-logs.json index ea9b78d..4408bf0 100644 --- a/assets/loki/dashboards/loki-logs.json +++ b/assets/loki/dashboards/loki-logs.json @@ -77,7 +77,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -114,6 +114,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -164,7 +169,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -236,7 +241,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[5m]))", + "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[$__rate_interval]))", "refId": "A" } ], @@ -250,7 +255,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -287,6 +292,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -336,7 +346,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -373,6 +383,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "binBps" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -408,7 +423,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))", + "expr": "sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[$__rate_interval]))", "refId": "A" } ], @@ -422,7 +437,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -459,6 +474,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "binBps" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -494,7 +514,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))", + "expr": "sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[$__rate_interval]))", "refId": "A" } ], @@ -508,7 +528,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -595,7 +615,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -632,6 +652,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "ops" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -667,7 +692,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"$deployment.*\", exported_pod=~\"$pod\", container=~\"$container\"}[5m])) by (level)", + "expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"$deployment.*\", exported_pod=~\"$pod\", container=~\"$container\"}[$__rate_interval])) by (level)", "legendFormat": "{{level}}", "refId": "A" } @@ -682,7 +707,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -719,6 +744,11 @@ "dashLength": 10, "dashes": false, "datasource": "$loki_datasource", + "fieldConfig": { + "defaults": { + "unit": "ops" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -771,7 +801,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\" } |logfmt| level=\"$level\" |= \"$filter\" [5m])) by (level)", + "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\" } |logfmt| level=\"$level\" |= \"$filter\" | __error__=\"\" [$__auto])) by (level)", "intervalFactor": 3, "legendFormat": "{{level}}", "refId": "A" @@ -787,7 +817,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -861,7 +891,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/loki/dashboards/loki-mixin-recording-rules.json b/assets/loki/dashboards/loki-mixin-recording-rules.json index b917b56..4ee5406 100644 --- a/assets/loki/dashboards/loki-mixin-recording-rules.json +++ b/assets/loki/dashboards/loki-mixin-recording-rules.json @@ -300,7 +300,8 @@ "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [ ] }, @@ -600,7 +601,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/loki/dashboards/loki-operational.json b/assets/loki/dashboards/loki-operational.json index 480fc21..4b52a4d 100644 --- a/assets/loki/dashboards/loki-operational.json +++ b/assets/loki/dashboards/loki-operational.json @@ -87,7 +87,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", + "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", "legendFormat": "{{status}}", "refId": "A" } @@ -102,7 +102,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -183,7 +183,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/distributor\", route=~\"api_prom_push|loki_api_v1_push\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", + "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/distributor\", route=~\"api_prom_push|loki_api_v1_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "legendFormat": "{{status}}", "refId": "A" } @@ -198,7 +198,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -237,7 +237,8 @@ "datasource": "$datasource", "fieldConfig": { "defaults": { - "custom": { } + "custom": { }, + "unit": "ops" }, "overrides": [ ] }, @@ -294,7 +295,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -374,7 +375,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant))", + "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (tenant))", "legendFormat": "{{tenant}}", "refId": "A" } @@ -389,7 +390,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -428,7 +429,8 @@ "datasource": "$datasource", "fieldConfig": { "defaults": { - "custom": { } + "custom": { }, + "unit": "MBs" }, "overrides": [ ] }, @@ -470,7 +472,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant)) / 1024 / 1024", + "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (tenant)) / 1024 / 1024", "legendFormat": "{{tenant}}", "refId": "A" } @@ -485,7 +487,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -581,7 +583,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -620,7 +622,8 @@ "datasource": "$datasource", "fieldConfig": { "defaults": { - "custom": { } + "custom": { }, + "unit": "ms" }, "overrides": [ ] }, @@ -687,7 +690,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -726,7 +729,8 @@ "datasource": "$datasource", "fieldConfig": { "defaults": { - "custom": { } + "custom": { }, + "unit": "ms" }, "overrides": [ ] }, @@ -793,7 +797,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -890,7 +894,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -930,7 +934,8 @@ "datasource": "$datasource", "fieldConfig": { "defaults": { - "custom": { } + "custom": { }, + "unit": "ms" }, "overrides": [ ] }, @@ -999,7 +1004,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -1096,7 +1101,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -1136,7 +1141,8 @@ "datasource": "$datasource", "fieldConfig": { "defaults": { - "custom": { } + "custom": { }, + "unit": "ms" }, "overrides": [ ] }, @@ -1205,7 +1211,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -1244,7 +1250,8 @@ "datasource": "$datasource", "fieldConfig": { "defaults": { - "custom": { } + "custom": { }, + "unit": "ms" }, "overrides": [ ] }, @@ -1311,7 +1318,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -1408,7 +1415,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -1449,7 +1456,8 @@ "description": "", "fieldConfig": { "defaults": { - "custom": { } + "custom": { }, + "unit": "ms" }, "overrides": [ ] }, @@ -1516,7 +1524,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -1613,7 +1621,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -1698,7 +1706,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])))", + "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])))", "interval": "", "legendFormat": "{{ tenant }} - {{ reason }}", "refId": "A" @@ -1714,7 +1722,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -1746,9 +1754,93 @@ } }, { - "columns": [ ], "datasource": "$datasource", - "fontSize": "100%", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "right", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "tenant" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reason" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + } + ] + }, "gridPos": { "h": 8, "w": 12, @@ -1756,74 +1848,23 @@ "y": 27 }, "id": 113, - "pageSize": null, - "panels": [ ], - "showHeader": true, - "sort": { - "col": 3, - "desc": true + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "tenant", - "thresholds": [ ], - "type": "string", - "unit": "short" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "reason", - "thresholds": [ ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "align": "right", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ ], - "type": "number", - "unit": "short" - } - ], + "panels": [ ], + "pluginVersion": "10.4.0", "targets": [ { - "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])[$__range:1m])))", + "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])[$__range:$__rate_interval])))", "format": "table", "instant": true, "interval": "", @@ -1831,11 +1872,16 @@ "refId": "A" } ], - "timeFrom": null, - "timeShift": null, "title": "Discarded Lines Per Interval", - "transform": "table", - "type": "table-old" + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [ ] + } + } + ], + "type": "table" } ], "targets": [ ], @@ -1895,9 +1941,9 @@ "steppedLine": false, "targets": [ { - "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"distributor.*\"}", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"distributor.*\"}[$__rate_interval]))", "intervalFactor": 3, - "legendFormat": "{{pod}}-{{container}}", + "legendFormat": "{{pod}}", "refId": "A" } ], @@ -1911,7 +1957,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -1948,6 +1994,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "binBps" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -2003,7 +2054,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -2081,7 +2132,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\"} | logfmt | level=\"error\"[1m]))", + "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\"} | logfmt | level=\"error\"[$__auto]))", "refId": "A" } ], @@ -2095,7 +2146,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -2212,7 +2263,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -2249,6 +2300,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "binBps" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -2285,7 +2341,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", + "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "intervalFactor": 1, "legendFormat": "{{pod}}", "refId": "A" @@ -2301,7 +2357,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -2338,6 +2394,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "binBps" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -2374,7 +2435,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", + "expr": "sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "intervalFactor": 1, "legendFormat": "{{pod}}", "refId": "A" @@ -2390,7 +2451,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -2427,6 +2488,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "ops" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -2463,7 +2529,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", + "expr": "sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "intervalFactor": 1, "legendFormat": "{{pod}}", "refId": "A" @@ -2479,7 +2545,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -2568,9 +2634,9 @@ "steppedLine": false, "targets": [ { - "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"ingester.*\"}", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"ingester.*\"}[$__rate_interval]))", "intervalFactor": 3, - "legendFormat": "{{pod}}-{{container}}", + "legendFormat": "{{pod}}", "refId": "A" } ], @@ -2584,7 +2650,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -2621,6 +2687,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "binBps" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -2676,7 +2747,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -2754,7 +2825,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/ingester.*\"} | logfmt | level=\"error\"[1m]))", + "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/ingester.*\"} | logfmt | level=\"error\"[$__auto]))", "refId": "A" } ], @@ -2768,7 +2839,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -2885,7 +2956,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -2992,7 +3063,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -3067,7 +3138,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster=\"$cluster\",job=~\"($namespace)/ingester.*\"}[1m]) > 0))", + "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster=\"$cluster\",job=~\"($namespace)/ingester.*\"}[$__rate_interval]) > 0))", "interval": "", "legendFormat": "{{ tenant }}", "refId": "A" @@ -3083,7 +3154,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -3177,13 +3248,13 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\",job=~\"($namespace)/ingester.*\"}[1m]))", + "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\",job=~\"($namespace)/ingester.*\"}[$__rate_interval]))", "interval": "", "legendFormat": "Chunks", "refId": "A" }, { - "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", job=~\"($namespace)/ingester.*\"}[1m]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"($namespace)/ingester.*\"}[1m])) < 1", + "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", job=~\"($namespace)/ingester.*\"}[$__rate_interval]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"($namespace)/ingester.*\"}[$__rate_interval])) < 1", "interval": "", "legendFormat": "De-Dupe Ratio", "refId": "B" @@ -3199,7 +3270,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -3261,7 +3332,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\",job=~\"($namespace)/ingester.*\"}[1m])) by (le)", + "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\",job=~\"($namespace)/ingester.*\"}[$__rate_interval])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -3352,7 +3423,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -3416,7 +3487,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"($namespace)/ingester.*\"}[1m]))", + "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"($namespace)/ingester.*\"}[$__rate_interval]))", "format": "heatmap", "instant": false, "interval": "", @@ -3508,9 +3579,9 @@ "steppedLine": false, "targets": [ { - "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"querier.*\"}", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"querier.*\"}[$__rate_interval]))", "intervalFactor": 3, - "legendFormat": "{{pod}}-{{container}}", + "legendFormat": "{{pod}}", "refId": "A" } ], @@ -3524,7 +3595,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -3561,6 +3632,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "binBps" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -3616,7 +3692,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -3694,7 +3770,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\"} | logfmt | level=\"error\"[1m]))", + "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\"} | logfmt | level=\"error\"[$__auto]))", "refId": "A" } ], @@ -3708,7 +3784,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -3825,7 +3901,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -3878,6 +3954,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -3917,19 +3998,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", + "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", "intervalFactor": 1, "legendFormat": "{{container}}: .99-{{method}}-{{name}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", + "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", "hide": false, "legendFormat": "{{container}}: .9-{{method}}-{{name}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", + "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", "hide": false, "legendFormat": "{{container}}: .5-{{method}}-{{name}}", "refId": "C" @@ -3945,7 +4026,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -4021,7 +4102,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, method, name, container)", + "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, method, name, container)", "intervalFactor": 1, "legendFormat": "{{container}}: {{status_code}}-{{method}}-{{name}}", "refId": "A" @@ -4037,7 +4118,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -4090,6 +4171,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -4129,19 +4215,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -4157,7 +4243,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -4194,6 +4280,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "ops" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -4233,7 +4324,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, status_code, method)", + "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, status_code, method)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -4249,7 +4340,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -4302,6 +4393,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -4339,17 +4435,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".9", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", "refId": "C" } ], @@ -4363,7 +4459,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -4400,6 +4496,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -4437,20 +4538,20 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", "interval": "", "intervalFactor": 1, "legendFormat": "99%", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", "interval": "", "legendFormat": "90%", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", "interval": "", "legendFormat": "50%", "refId": "C" @@ -4466,7 +4567,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -4503,6 +4604,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -4540,20 +4646,20 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", "interval": "", "intervalFactor": 1, "legendFormat": "99%", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", "interval": "", "legendFormat": "90%", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", "interval": "", "legendFormat": "50%", "refId": "C" @@ -4569,7 +4675,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -4606,6 +4712,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -4643,17 +4754,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".9", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", "refId": "C" } ], @@ -4667,7 +4778,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -4704,6 +4815,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "ops" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -4741,7 +4857,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -4757,7 +4873,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -4794,6 +4910,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "ops" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -4831,7 +4952,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -4847,7 +4968,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -4884,6 +5005,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "ops" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -4921,7 +5047,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -4937,7 +5063,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -4974,6 +5100,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "ops" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -5011,7 +5142,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -5027,7 +5158,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -5080,6 +5211,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -5119,19 +5255,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -5147,7 +5283,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -5223,7 +5359,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -5239,7 +5375,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -5327,7 +5463,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(cortex_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", + "expr": "sum(rate(loki_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -5341,7 +5477,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -5413,7 +5549,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(cortex_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", + "expr": "sum(rate(loki_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -5427,7 +5563,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -5499,7 +5635,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(cortex_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", + "expr": "sum(rate(loki_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -5513,7 +5649,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -5585,7 +5721,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(cortex_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", + "expr": "sum(rate(loki_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -5599,7 +5735,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -5671,17 +5807,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", + "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", + "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", + "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", "legendFormat": ".5", "refId": "C" } @@ -5696,7 +5832,7 @@ "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -5733,6 +5869,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -5771,19 +5912,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -5799,7 +5940,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -5874,7 +6015,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(cortex_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "expr": "sum(rate(loki_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -5890,7 +6031,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -5943,6 +6084,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -5981,19 +6127,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -6009,7 +6155,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -6084,7 +6230,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -6100,7 +6246,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -6153,6 +6299,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -6191,19 +6342,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -6219,7 +6370,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -6294,7 +6445,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -6310,7 +6461,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -6363,6 +6514,11 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -6401,19 +6557,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -6429,7 +6585,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -6504,7 +6660,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -6520,7 +6676,7 @@ "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "timeseries", "xaxis": { "buckets": null, "mode": "time", @@ -6572,7 +6728,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/loki/dashboards/loki-reads-resources.json b/assets/loki/dashboards/loki-reads-resources.json index 205362c..bc3d000 100644 --- a/assets/loki/dashboards/loki-reads-resources.json +++ b/assets/loki/dashboards/loki-reads-resources.json @@ -27,280 +27,256 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 1, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\", resource=\"cpu\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 2, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\", resource=\"memory\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-frontend\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -315,280 +291,256 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 4, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\", resource=\"cpu\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 5, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\", resource=\"memory\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"query-scheduler\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -600,517 +552,400 @@ }, { "collapse": false, - "collapsed": false, + "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 7, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + }, + "span": 2, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\", resource=\"cpu\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 8, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + }, + "span": 2, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\", resource=\"memory\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 9, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/querier\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, - "gridPos": { }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, "targets": [ { - "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "expr": "sum by(instance, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Writes", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, - "gridPos": { }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, "targets": [ { - "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "expr": "sum by(instance,device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Reads", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 12, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, "targets": [ { - "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"querier.*\"})", + "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*querier.*\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*querier.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{persistentvolumeclaim}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Space Utilization", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1118,522 +953,404 @@ "repeatRowId": null, "showTitle": true, "title": "Querier", - "titleSize": "h6", - "type": "row" + "titleSize": "h6" }, { "collapse": false, - "collapsed": false, + "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 13, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + }, + "span": 2, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\", resource=\"cpu\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 14, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + }, + "span": 2, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\", resource=\"memory\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 15, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/index-gateway\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, - "gridPos": { }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, "targets": [ { - "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "expr": "sum by(instance, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Writes", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, - "gridPos": { }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, "targets": [ { - "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "expr": "sum by(instance, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Reads", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 18, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, "targets": [ { - "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"index-gateway.*\"})", + "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*index-gateway.*\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*index-gateway.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{persistentvolumeclaim}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Space Utilization", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1641,288 +1358,668 @@ "repeatRowId": null, "showTitle": true, "title": "Index Gateway", - "titleSize": "h6", - "type": "row" + "titleSize": "h6" }, { "collapse": false, "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 19, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, + }, + "span": 2, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\", resource=\"cpu\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"cpu\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"})", + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 20, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, + }, + "span": 2, "targets": [ { - "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"})", + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"memory\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"} > 0)", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 21, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, "targets": [ { - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/ingester.+\"})", + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] }, - "yaxes": [ + "id": 22, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, + "targets": [ { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "expr": "sum by(instance, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "format": "time_series", + "legendFormat": "{{pod}} - {{device}}", + "legendLink": null + } + ], + "title": "Disk Writes", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, + "id": 23, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, + "targets": [ + { + "expr": "sum by(instance, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "format": "time_series", + "legendFormat": "{{pod}} - {{device}}", + "legendLink": null + } + ], + "title": "Disk Reads", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "id": 24, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, + "targets": [ + { + "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*bloom-gateway.*\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*bloom-gateway.*\"})", + "format": "time_series", + "legendFormat": "{{persistentvolumeclaim}}", + "legendLink": null + } + ], + "title": "Disk Space Utilization", + "type": "timeseries" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bloom Gateway", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 25, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\", resource=\"cpu\"} > 0)", + "format": "time_series", + "legendFormat": "request", + "legendLink": null + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"})", + "format": "time_series", + "legendFormat": "limit", + "legendLink": null } - ] + ], + "title": "CPU", + "tooltip": { + "sort": 2 + }, + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 26, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"})", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\", resource=\"memory\"} > 0)", + "format": "time_series", + "legendFormat": "request", + "legendLink": null + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"} > 0)", + "format": "time_series", + "legendFormat": "limit", + "legendLink": null + } + ], + "title": "Memory (workingset)", + "tooltip": { + "sort": 2 + }, + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "id": 27, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/ingester.+\"})", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + } + ], + "title": "Memory (go heap inuse)", + "tooltip": { + "sort": 2 + }, + "type": "timeseries" } ], "repeat": null, @@ -1934,363 +2031,306 @@ }, { "collapse": false, - "collapsed": false, + "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 28, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 3, "targets": [ { - "expr": "sum by(pod) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"})", + "expr": "sum by(pod) (loki_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}) or sum by(pod) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Rules", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 29, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + }, + "span": 3, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\", resource=\"cpu\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 30, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + }, + "span": 3, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\", resource=\"memory\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 31, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 3, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2298,8 +2338,7 @@ "repeatRowId": null, "showTitle": true, "title": "Ruler", - "titleSize": "h6", - "type": "row" + "titleSize": "h6" } ], "schemaVersion": 14, @@ -2315,7 +2354,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/loki/dashboards/loki-reads.json b/assets/loki/dashboards/loki-reads.json index ef1acc9..b592131 100644 --- a/assets/loki/dashboards/loki-reads.json +++ b/assets/loki/dashboards/loki-reads.json @@ -33,156 +33,256 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/query-frontend\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/query-frontend\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ route }} 99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "{{ route }} 99th percentile", + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"})) * 1e3", + "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ route }} 50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "{{ route }} 50th percentile", + "refId": "B" }, { - "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}) by (route) ", + "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route) ", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{ route }} Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -203,89 +303,55 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { - "custom": { - "fillOpacity": 50, - "showPoints": "never", - "stacking": { - "group": "A", - "mode": "normal" - } - } + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "fill": 1, "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99,\n sum(\n rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval])\n ) by (pod, le)\n )\n", - "instant": false, + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval])) by (le,pod)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, "legendFormat": "__auto", - "range": true, - "refId": "A" + "refId": "A", + "step": 10 } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per Pod Latency (p99)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -306,156 +372,256 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/querier\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/querier\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ route }} 99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "{{ route }} 99th percentile", + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"})) * 1e3", + "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ route }} 50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "{{ route }} 50th percentile", + "refId": "B" }, { - "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}) by (route) ", + "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route) ", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{ route }} Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -476,89 +642,55 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { - "custom": { - "fillOpacity": 50, - "showPoints": "never", - "stacking": { - "group": "A", - "mode": "normal" - } - } + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "fill": 1, "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99,\n sum(\n rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval])\n ) by (pod, le)\n )\n", - "instant": false, + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval])) by (le,pod)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, "legendFormat": "__auto", - "range": true, - "refId": "A" + "refId": "A", + "step": 10 } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per Pod Latency (p99)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -579,156 +711,256 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/ingester\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/ingester\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 8, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ route }} 99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "{{ route }} 99th percentile", + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"})) * 1e3", + "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ route }} 50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "{{ route }} 50th percentile", + "refId": "B" }, { - "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}) by (route) ", + "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) ", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{ route }} Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -749,89 +981,55 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { - "custom": { - "fillOpacity": 50, - "showPoints": "never", - "stacking": { - "group": "A", - "mode": "normal" - } - } + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "fill": 1, "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99,\n sum(\n rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])\n ) by (pod, le)\n )\n", - "instant": false, + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval])) by (le,pod)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, "legendFormat": "__auto", - "range": true, - "refId": "A" + "refId": "A", + "step": 10 } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per Pod Latency (p99)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -852,156 +1050,256 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/ingester-zone.*\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/ingester-zone.*\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 11, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ route }} 99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "{{ route }} 99th percentile", + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"})) * 1e3", + "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ route }} 50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "{{ route }} 50th percentile", + "refId": "B" }, { - "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}) by (route) ", + "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) ", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{ route }} Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1022,89 +1320,55 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { - "custom": { - "fillOpacity": 50, - "showPoints": "never", - "stacking": { - "group": "A", - "mode": "normal" - } - } + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "fill": 1, "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99,\n sum(\n rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])\n ) by (pod, le)\n )\n", - "instant": false, + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval])) by (le,pod)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, "legendFormat": "__auto", - "range": true, - "refId": "A" + "refId": "A", + "step": 10 } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per Pod Latency (p99)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1125,156 +1389,256 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_index_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/index-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 14, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/index-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "{{ route }} 99th percentile", + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/index-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "{{ route }} 50th percentile", + "refId": "B" }, { - "expr": "sum(rate(loki_index_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval])) * 1e3 / sum(rate(loki_index_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval]))", + "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/index-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/index-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) ", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "refId": "C", - "step": 10 + "legendFormat": "{{ route }} Average", + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1295,96 +1659,62 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { - "custom": { - "fillOpacity": 50, - "showPoints": "never", - "stacking": { - "group": "A", - "mode": "normal" - } - } + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "fill": 1, "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99,\n sum(\n rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval])\n ) by (pod, le)\n )\n", - "instant": false, + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/index-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval])) by (le,pod)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, "legendFormat": "__auto", - "range": true, - "refId": "A" + "refId": "A", + "step": 10 } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per Pod Latency (p99)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Index", + "title": "Index Gateway", "titleSize": "h6" }, { @@ -1398,156 +1728,256 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/bloom-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 17, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "{{ route }} 99th percentile", + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "{{ route }} 50th percentile", + "refId": "B" }, { - "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval]))", + "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) ", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "refId": "C", - "step": 10 + "legendFormat": "{{ route }} Average", + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1568,74 +1998,328 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { - "custom": { - "fillOpacity": 50, - "showPoints": "never", - "stacking": { - "group": "A", - "mode": "normal" - } - } + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "fill": 1, "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99,\n sum(\n rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval])\n ) by (pod, le)\n )\n", - "instant": false, + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}[$__rate_interval])) by (le,pod)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, "legendFormat": "__auto", - "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Per Pod Latency (p99)", + "type": "timeseries" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bloom Gateway", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, + "fill": 10, + "id": 19, + "linewidth": 0, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "stack": true, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_index_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "legendFormat": "{{status}}", "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Per Pod Latency (p99)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + "title": "QPS", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "id": 20, + "links": [ ], + "nullPointMode": "null as zero", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, + "span": 4, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "legendFormat": "99th Percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "legendFormat": "50th Percentile", + "refId": "B" + }, + { + "expr": "sum(rate(loki_index_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval])) * 1e3 / sum(rate(loki_index_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "Average", + "refId": "C" + } + ], + "title": "Latency", + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1651,13 +2335,403 @@ "show": false } ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, + "id": 21, + "links": [ ], + "nullPointMode": "null as zero", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/querier\", operation!=\"index_chunk\"}[$__rate_interval])) by (le,pod)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "__auto", + "refId": "A", + "step": 10 + } + ], + "title": "Per Pod Latency (p99)", + "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "BoltDB Shipper", + "title": "TSBD Index", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, + "fill": 10, + "id": 22, + "linewidth": 0, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "stack": true, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "QPS", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, + "id": 23, + "links": [ ], + "nullPointMode": "null as zero", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "legendFormat": "99th Percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "legendFormat": "50th Percentile", + "refId": "B" + }, + { + "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "Average", + "refId": "C" + } + ], + "title": "Latency", + "type": "timeseries", + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, + "id": 24, + "links": [ ], + "nullPointMode": "null as zero", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(querier|index-gateway)\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le,pod)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "__auto", + "refId": "A", + "step": 10 + } + ], + "title": "Per Pod Latency (p99)", + "type": "timeseries" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "BoltDB Index", "titleSize": "h6" } ], @@ -1674,7 +2748,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/loki/dashboards/loki-retention.json b/assets/loki/dashboards/loki-retention.json index b848b82..8223834 100644 --- a/assets/loki/dashboards/loki-retention.json +++ b/assets/loki/dashboards/loki-retention.json @@ -27,280 +27,256 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 1, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\", resource=\"cpu\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 2, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\", resource=\"memory\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -375,7 +351,7 @@ "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, - "span": 4, + "span": 6, "stack": false, "steppedLine": false, "targets": [ @@ -389,7 +365,7 @@ "thresholds": [ ], "timeFrom": null, "timeShift": null, - "title": "Last Compact and Mark Operation Success", + "title": "Last Compact Tables Operation Success", "tooltip": { "shared": true, "sort": 2, @@ -423,163 +399,164 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 6, "targets": [ { "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}", "format": "time_series", - "intervalFactor": 2, "legendFormat": "duration", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Compact and Mark Operations Duration", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status)(rate(loki_boltdb_shipper_compact_tables_operation_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{success}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Compact and Mark Operations Per Status", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "title": "Compact Tables Operations Duration", + "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Compact and Mark", + "title": "Compaction", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "id": 6, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 6, + "targets": [ + { + "expr": "sum(increase(loki_compactor_skipped_compacting_locked_table_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__range]))", + "format": "time_series", + "legendFormat": "{{table_name}}", + "legendLink": null + } + ], + "title": "Number of times Tables were skipped during Compaction", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "id": 7, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 6, + "targets": [ + { + "expr": "sum by (status)(rate(loki_boltdb_shipper_compact_tables_operation_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "{{success}}", + "legendLink": null + } + ], + "title": "Compact Tables Operations Per Status", + "type": "timeseries" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "", "titleSize": "h6" }, { @@ -592,83 +569,26 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "count by(action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{action}}", - "legendLink": null, - "step": 10 + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dateTimeFromNow" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Processed Tables Per Action", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, + "fill": 1, "id": 8, "legend": { "avg": false, @@ -680,9 +600,24 @@ "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { }, + "textMode": "auto" + }, "percentage": false, "pointradius": 5, "points": false, @@ -690,28 +625,26 @@ "seriesOverrides": [ ], "spaceLength": 10, "span": 4, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "count by(table,action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\" , action=~\"modified|deleted\"})", + "expr": "loki_compactor_apply_retention_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"} * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{table}}-{{action}}", - "legendLink": null, - "step": 10 + "instant": true, + "refId": "A" } ], "thresholds": [ ], "timeFrom": null, "timeShift": null, - "title": "Modified Tables", + "title": "Last Mark Operation Success", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, - "type": "graph", + "type": "stat", "xaxis": { "buckets": null, "mode": "time", @@ -739,80 +672,251 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 0, + "id": 9, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "loki_compactor_apply_retention_operation_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}", + "format": "time_series", + "legendFormat": "duration", + "legendLink": null + } + ], + "title": "Mark Operations Duration", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "id": 10, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum by (status)(rate(loki_compactor_apply_retention_operation_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "{{success}}", + "legendLink": null + } + ], + "title": "Mark Operations Per Status", + "type": "timeseries" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Retention", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "id": 11, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "count by(action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"})", + "format": "time_series", + "legendFormat": "{{action}}", + "legendLink": null + } + ], + "title": "Processed Tables Per Action", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "id": 12, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "count by(table,action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\" , action=~\"modified|deleted\"})", + "format": "time_series", + "legendFormat": "{{table}}-{{action}}", + "legendLink": null + } + ], + "title": "Modified Tables", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "id": 13, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by (table)(rate(loki_boltdb_shipper_retention_marker_count_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) >0", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{table}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Marks Creation Rate Per Table", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -827,154 +931,113 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "format": "short", - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "format": "short", + "id": 14, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum (increase(loki_boltdb_shipper_retention_marker_count_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[24h]))", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Marked Chunks (24h)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 15, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Mark Table Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1007,154 +1070,113 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "format": "short", - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "format": "short", + "id": 16, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum (increase(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[24h]))", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Delete Chunks (24h)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 17, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Delete Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1187,232 +1209,145 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 18, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "time() - (loki_boltdb_shipper_retention_sweeper_marker_file_processing_current_time{cluster=~\"$cluster\", namespace=~\"$namespace\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "lag", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Sweeper Lag", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 19, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(loki_boltdb_shipper_retention_sweeper_marker_files_current{cluster=~\"$cluster\", namespace=~\"$namespace\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "count", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Marks Files to Process", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 20, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by (status)(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Delete Rate Per Status", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1428,7 +1363,7 @@ "panels": [ { "datasource": "$loki_datasource", - "id": 17, + "id": 21, "span": 12, "targets": [ { @@ -1461,7 +1396,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/loki/dashboards/loki-writes-resources.json b/assets/loki/dashboards/loki-writes-resources.json index f934efe..5bd3d74 100644 --- a/assets/loki/dashboards/loki-writes-resources.json +++ b/assets/loki/dashboards/loki-writes-resources.json @@ -27,280 +27,256 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 1, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"distributor\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"distributor\", resource=\"cpu\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"distributor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"distributor\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 2, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"distributor\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"distributor\", resource=\"memory\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"distributor\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -312,592 +288,450 @@ }, { "collapse": false, - "collapsed": false, + "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 4, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 1, "targets": [ { "expr": "sum by(pod) (loki_ingester_memory_streams{cluster=~\"$cluster\", job=~\"($namespace)/ingester.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "In-memory streams", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 5, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + }, + "span": 1, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\", resource=\"cpu\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 6, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + }, + "span": 1, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\", resource=\"memory\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 1, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/ingester.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, - "gridPos": { }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 1, "targets": [ { - "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "expr": "sum by(instance, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Writes", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, - "gridPos": { }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 1, "targets": [ { - "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "expr": "sum by(instance, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Reads", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "gridPos": { }, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 10, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 1, "targets": [ { - "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"ingester.*.*\"})", + "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*ingester.*.*\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*ingester.*.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{persistentvolumeclaim}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Space Utilization", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -905,8 +739,7 @@ "repeatRowId": null, "showTitle": true, "title": "Ingester", - "titleSize": "h6", - "type": "row" + "titleSize": "h6" } ], "schemaVersion": 14, @@ -922,7 +755,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/loki/dashboards/loki-writes.json b/assets/loki/dashboards/loki-writes.json index fcd4040..9204ae6 100644 --- a/assets/loki/dashboards/loki-writes.json +++ b/assets/loki/dashboards/loki-writes.json @@ -33,156 +33,256 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/distributor\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/distributor\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/distributor\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"})) * 1e3", + "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/distributor\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { - "expr": "1e3 * sum(cluster_job:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"}) / sum(cluster_job:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"})", + "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/distributor\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"}) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/distributor\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -201,6 +301,57 @@ "show": false } ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, + "id": 3, + "links": [ ], + "nullPointMode": "null as zero", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/distributor\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"}[$__rate_interval])) by (le,pod)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "__auto", + "refId": "A", + "step": 10 + } + ], + "title": "Per Pod Latency (p99)", + "type": "timeseries" } ], "repeat": null, @@ -215,76 +366,105 @@ "height": "250px", "panels": [ { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 0, + "id": 4, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/ingester-zone.*\", route=\"/logproto.Pusher/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum (rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/distributor\",}[$__rate_interval])) / sum(rate(loki_distributor_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/distributor\",}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "legendFormat": "bytes", + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + "title": "Per Total Received Bytes", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "id": 5, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, + "span": 6, + "stack": true, + "targets": [ + { + "expr": "sum by (tenant) (rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/distributor\",}[$__rate_interval])) / ignoring(tenant) group_left sum(rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/distributor\",}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "{{tenant}}", + "legendLink": null + } + ], + "title": "Per Tenant", + "type": "timeseries", "yaxes": [ { "format": "short", "label": null, "logBase": 1, - "max": null, + "max": 1, "min": 0, "show": true }, @@ -292,85 +472,281 @@ "format": "short", "label": null, "logBase": 1, - "max": null, + "max": 1, "min": null, "show": false } ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Distributor - Structured Metadata", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", + "error": "#E24D42", + "success": "#7EB26D" + }, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, + "fill": 10, + "id": 6, + "linewidth": 0, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "stack": true, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/ingester-zone.*\", route=\"/logproto.Pusher/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "QPS", + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=\"/logproto.Pusher/Push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=\"/logproto.Pusher/Push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=\"/logproto.Pusher/Push\"}) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=\"/logproto.Pusher/Push\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -389,6 +765,57 @@ "show": false } ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, + "id": 8, + "links": [ ], + "nullPointMode": "null as zero", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester-zone.*\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (le,pod)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "__auto", + "refId": "A", + "step": 10 + } + ], + "title": "Per Pod Latency (p99)", + "type": "timeseries" } ], "repeat": null, @@ -409,156 +836,256 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] }, - "lines": true, + "fill": 10, + "id": 9, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 10, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\"}) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -577,6 +1104,57 @@ "show": false } ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, + "id": 11, + "links": [ ], + "nullPointMode": "null as zero", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (le,pod)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "__auto", + "refId": "A", + "step": 10 + } + ], + "title": "Per Pod Latency (p99)", + "type": "timeseries" } ], "repeat": null, @@ -597,156 +1175,256 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] }, - "lines": true, + "fill": 10, + "id": 12, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_index_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/ingester.*\", operation=\"index_chunk\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_index_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/ingester.*\", operation=\"index_chunk\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 13, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/ingester.*\", operation=\"index_chunk\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/ingester.*\", operation=\"index_chunk\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(loki_index_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/ingester.*\", operation=\"index_chunk\"}[$__rate_interval])) * 1e3 / sum(rate(loki_index_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/ingester.*\", operation=\"index_chunk\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -765,6 +1443,57 @@ "show": false } ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, + "id": 14, + "links": [ ], + "nullPointMode": "null as zero", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/ingester.*\", operation=\"index_chunk\"}[$__rate_interval])) by (le,pod)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "__auto", + "refId": "A", + "step": 10 + } + ], + "title": "Per Pod Latency (p99)", + "type": "timeseries" } ], "repeat": null, @@ -785,156 +1514,256 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] }, - "lines": true, + "fill": 10, + "id": 15, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/ingester\", operation=\"WRITE\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/ingester\", operation=\"WRITE\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 16, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/ingester\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/ingester\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/ingester\", operation=\"WRITE\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/ingester\", operation=\"WRITE\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -953,13 +1782,64 @@ "show": false } ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, + "id": 17, + "links": [ ], + "nullPointMode": "null as zero", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/ingester\", operation=\"WRITE\"}[$__rate_interval])) by (le,pod)) * 1e3", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "__auto", + "refId": "A", + "step": 10 + } + ], + "title": "Per Pod Latency (p99)", + "type": "timeseries" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "BoltDB Shipper", + "title": "BoltDB Index", "titleSize": "h6" } ], @@ -976,7 +1856,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/assets/memcached/alerts.yaml b/assets/memcached/alerts.yaml index 94cde50..6578531 100644 --- a/assets/memcached/alerts.yaml +++ b/assets/memcached/alerts.yaml @@ -31,3 +31,14 @@ groups: for: 15m labels: severity: critical + - alert: MemcachedOutOfMemoryErrors + annotations: + description: Memcached instance {{ $labels.job }} / {{ $labels.instance }} has + OutOfMemory errors for at least 15 minutes, current rate is {{ printf "%0.0f" + $value }} + summary: Memcached has OutOfMemory errors. + expr: | + sum without (slab) (rate(memcached_slab_items_outofmemory_total[5m])) > 0 + for: 15m + labels: + severity: warning diff --git a/assets/mongodb/alerts.yaml b/assets/mongodb/alerts.yaml index 7b66522..c728ff5 100644 --- a/assets/mongodb/alerts.yaml +++ b/assets/mongodb/alerts.yaml @@ -9,7 +9,15 @@ groups: LABELS = {{ $labels }} summary: MongoDB Instance is Down. expr: mongodb_up == 0 - for: 0m + for: 5m + labels: + severity: critical + - alert: MongodbReplicaMemberUnhealthy + annotations: + description: Mongodb replica member unhealthy (instance {{ $labels.instance + }}) + summary: MongoDB replica member unhealthy. + expr: mongodb_mongod_replset_member_health == 0 labels: severity: critical - alert: MongodbReplicationLag @@ -19,9 +27,8 @@ groups: VALUE = {{ $value }} LABELS = {{ $labels }} summary: MongoDB replication lag is exceeding the threshold. - expr: mongodb_mongod_replset_member_optime_date{state="PRIMARY"} - ON (set) mongodb_mongod_replset_member_optime_date{state="SECONDARY"} - > 10 - for: 0m + expr: mongodb_mongod_replset_member_replication_lag{state="SECONDARY"} > 60 + for: 5m labels: severity: critical - alert: MongodbReplicationHeadroom @@ -34,7 +41,7 @@ groups: expr: (avg(mongodb_mongod_replset_oplog_tail_timestamp - mongodb_mongod_replset_oplog_head_timestamp) - (avg(mongodb_mongod_replset_member_optime_date{state="PRIMARY"}) - avg(mongodb_mongod_replset_member_optime_date{state="SECONDARY"}))) <= 0 - for: 0m + for: 5m labels: severity: critical - alert: MongodbNumberCursorsOpen @@ -83,3 +90,25 @@ groups: for: 2m labels: severity: warning + - alert: MongodbReadRequestsQueueingUp + annotations: + description: |- + MongoDB requests are queuing up + VALUE = {{ $value }} + LABELS = {{ $labels }} + summary: MongoDB read requests queuing up. + expr: delta(mongodb_mongod_global_lock_current_queue{type="reader"}[1m]) > 0 + for: 5m + labels: + severity: warning + - alert: MongodbWriteRequestsQueueingUp + annotations: + description: |- + MongoDB write requests are queueing up + VALUE = {{ $value }} + LABELS = {{ $labels }} + summary: MongoDB write requests queuing up. + expr: delta(mongodb_mongod_global_lock_current_queue{type="writer"}[1m]) > 0 + for: 5m + labels: + severity: warning diff --git a/assets/mongodb/dashboards/MongoDB_Cluster.json b/assets/mongodb/dashboards/MongoDB_Cluster.json index 0722349..65747e0 100644 --- a/assets/mongodb/dashboards/MongoDB_Cluster.json +++ b/assets/mongodb/dashboards/MongoDB_Cluster.json @@ -1591,7 +1591,7 @@ "refId": "A" } ], - "title": "Currrent Connections per Instance", + "title": "Current Connections per Instance", "type": "timeseries" }, { @@ -1672,7 +1672,7 @@ "refId": "A" } ], - "title": "Currrent Connections per Instance", + "title": "Current Connections per Instance", "type": "timeseries" }, { @@ -1834,7 +1834,7 @@ "refId": "A" } ], - "title": "Currrent Connections per Shard", + "title": "Current Connections per Shard", "type": "timeseries" }, { diff --git a/assets/nginx/dashboards/nginx-overview.json b/assets/nginx/dashboards/nginx-logs.json similarity index 70% rename from assets/nginx/dashboards/nginx-overview.json rename to assets/nginx/dashboards/nginx-logs.json index 1b1000b..f1c9a1b 100644 --- a/assets/nginx/dashboards/nginx-overview.json +++ b/assets/nginx/dashboards/nginx-logs.json @@ -1,62 +1,5 @@ { - "__elements": { }, "__inputs": [ ], - "__requires": [ - { - "id": "geomap", - "name": "Geomap", - "type": "panel", - "version": "" - }, - { - "id": "grafana", - "name": "Grafana", - "type": "grafana", - "version": "10.0.1-cloud.3.f250259e" - }, - { - "id": "graphite", - "name": "Graphite", - "type": "datasource", - "version": "1.0.0" - }, - { - "id": "logs", - "name": "Logs", - "type": "panel", - "version": "" - }, - { - "id": "loki", - "name": "Loki", - "type": "datasource", - "version": "1.0.0" - }, - { - "id": "stat", - "name": "Stat", - "type": "panel", - "version": "" - }, - { - "id": "table", - "name": "Table", - "type": "panel", - "version": "" - }, - { - "id": "text", - "name": "Text", - "type": "panel", - "version": "" - }, - { - "id": "timeseries", - "name": "Time series", - "type": "panel", - "version": "" - } - ], "annotations": { "list": [ { @@ -79,12 +22,12 @@ } ] }, - "description": "Loki v2+ showcase using JSON NGINX access logs.", + "description": "NGINX Access Logs", "editable": true, "fiscalYearStartMonth": 0, "gnetId": 12559, "graphTooltip": 0, - "id": null, + "id": 724, "links": [ ], "liveNow": false, "panels": [ @@ -150,6 +93,7 @@ "graphMode": "area", "justifyMode": "center", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "sum" @@ -157,27 +101,29 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": { }, - "textMode": "value" + "textMode": "value", + "wideLayout": true }, - "pluginVersion": "10.0.1-cloud.3.f250259e", + "pluginVersion": "11.1.0-71799", "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "sum by(host) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"}[$__interval])) ", + "expr": "sum by(host) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"}[$__interval])) ", "legendFormat": "", "refId": "A" } ], "timeFrom": "24h", "title": "Total requests ", - "transformations": [ ], "type": "stat" }, { "datasource": { + "type": "loki", "uid": "$datasource" }, "description": "", @@ -217,6 +163,7 @@ "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "sum" @@ -224,24 +171,27 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": { }, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.0.1-cloud.3.f250259e", + "pluginVersion": "11.1.0-71799", "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "sum by (status) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | __error__=\"\" [$__interval]))", + "editorMode": "code", + "expr": "sum by (status) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | __error__=\"\" [$__interval]))", "instant": false, "legendFormat": "HTTP Status: {{status}}", + "queryType": "range", "range": true, "refId": "A" } ], "title": "Requests per status code", - "transformations": [ ], "type": "stat" }, { @@ -282,6 +232,7 @@ "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "sum" @@ -289,16 +240,18 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": { }, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.0.1-cloud.3.f250259e", + "pluginVersion": "11.1.0-71799", "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "bytes_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"}[$__interval])", + "expr": "bytes_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"}[$__interval])", "instant": true, "legendFormat": "$label_value", "range": false, @@ -345,6 +298,7 @@ "graphMode": "none", "justifyMode": "center", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "sum" @@ -352,16 +306,18 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": { }, - "textMode": "value" + "textMode": "value", + "wideLayout": true }, - "pluginVersion": "10.0.1-cloud.3.f250259e", + "pluginVersion": "11.1.0-71799", "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "sum by (host) (sum_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | unwrap body_bytes_sent | __error__=\"\" [$__interval]))", + "expr": "sum by (host) (sum_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | unwrap body_bytes_sent | __error__=\"\" [$__interval]))", "instant": true, "legendFormat": "Bytes sent", "range": false, @@ -369,10 +325,79 @@ } ], "title": "Total Bytes Sent", - "transformations": [ ], "type": "stat" }, - { }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "hideTimeOverride": true, + "id": 18, + "interval": "10m", + "maxDataPoints": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { }, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.1.0-71799", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(count_over_time(({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} |= \"Googlebot\")[$__interval])) / (sum(count_over_time(({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} != \"Googlebot\")[$__interval])) / 100)", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "timeFrom": "1h", + "title": "% of requests by Googlebot", + "type": "stat" + }, { "datasource": { "uid": "$datasource" @@ -409,6 +434,7 @@ "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "mean" @@ -416,16 +442,18 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": { }, - "textMode": "value" + "textMode": "value", + "wideLayout": true }, - "pluginVersion": "10.0.1-cloud.3.f250259e", + "pluginVersion": "11.1.0-71799", "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "count(sum by (remote_addr) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | __error__=\"\" [$__interval])))", + "expr": "count(sum by (remote_addr) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | __error__=\"\" [$__interval])))", "instant": true, "legendFormat": "", "range": false, @@ -434,7 +462,6 @@ ], "timeFrom": "5m", "title": "Realtime visitors ", - "transformations": [ ], "type": "stat" }, { @@ -475,6 +502,7 @@ "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "sum" @@ -482,16 +510,18 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": { }, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.0.1-cloud.3.f250259e", + "pluginVersion": "11.1.0-71799", "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"}[$__interval])", + "expr": "count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"}[$__interval])", "instant": true, "range": false, "refId": "A" @@ -539,13 +569,13 @@ }, "hideTimeOverride": true, "id": 19, - "links": [ ], "maxDataPoints": 1, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "max" @@ -553,16 +583,18 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": { }, - "textMode": "value" + "textMode": "value", + "wideLayout": true }, - "pluginVersion": "10.0.1-cloud.3.f250259e", + "pluginVersion": "11.1.0-71799", "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "sum(count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | status >= 500 |__error__=\"\"[$__interval])) / (sum(count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | __error__=\"\"[$__interval]))/ 100)", + "expr": "sum(count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | status >= 500 |__error__=\"\"[$__interval])) / (sum(count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | __error__=\"\"[$__interval]))/ 100)", "instant": false, "legendFormat": "", "range": true, @@ -573,75 +605,6 @@ "title": "% of 5xx requests ", "type": "stat" }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - }, - "unit": "percent" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 5 - }, - "hideTimeOverride": true, - "id": 18, - "interval": "10m", - "links": [ ], - "maxDataPoints": 1, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { }, - "textMode": "value" - }, - "pluginVersion": "10.0.1-cloud.3.f250259e", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(count_over_time(({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} |= \"Googlebot\")[$__interval])) / (sum(count_over_time(({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} != \"Googlebot\")[$__interval])) / 100)", - "instant": true, - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "timeFrom": "1h", - "title": "% of requests by Googlebot", - "type": "stat" - }, { "datasource": { "uid": "$datasource" @@ -750,13 +713,13 @@ "zoom": 2 } }, - "pluginVersion": "10.0.1-cloud.3.f250259e", + "pluginVersion": "11.1.0-71799", "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "sum by (geoip_country_code) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | geoip_country_code != \"\" | __error__=\"\" [$__interval]))", + "expr": "sum by (geoip_country_code) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | geoip_country_code != \"\" | __error__=\"\" [$__interval]))", "instant": false, "legendFormat": "{{geoip_country_code}}", "range": true, @@ -880,13 +843,13 @@ } ] }, - "pluginVersion": "10.0.1-cloud.3.f250259e", + "pluginVersion": "11.1.0-71799", "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "topk(10, sum by (geoip_country_code) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | geoip_country_code != \"\" and __error__=\"\" [$__interval])))", + "expr": "topk(10, sum by (geoip_country_code) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | geoip_country_code != \"\" and __error__=\"\" [$__interval])))", "instant": true, "legendFormat": "{{remote_addr}}", "range": false, @@ -949,7 +912,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | line_format \"➡️ {{.request_method}} {{.request_uri}} with HTTP status: {{.status}} \"", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | line_format \"➡️ {{.request_method}} {{.request_uri}} with HTTP status: {{.status}} \"", "legendFormat": "", "refId": "A" } @@ -983,6 +946,7 @@ }, { "datasource": { + "type": "loki", "uid": "$datasource" }, "description": "", @@ -992,6 +956,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1005,6 +970,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, @@ -1027,7 +993,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "#EAB839", @@ -1112,7 +1079,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "quantile_over_time(0.95,{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | unwrap request_time | __error__=\"\" [$__interval]) by (host)", + "expr": "quantile_over_time(0.95,{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | unwrap request_time | __error__=\"\" [$__interval]) by (host)", "legendFormat": "95th percentile", "refId": "C" }, @@ -1120,8 +1087,10 @@ "datasource": { "uid": "$datasource" }, - "expr": "max by (host) (max_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | unwrap request_time | __error__=\"\" [$__interval]))", + "editorMode": "code", + "expr": "max by (host) (max_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | unwrap request_time | __error__=\"\" [$__interval]))", "legendFormat": "max latency", + "queryType": "range", "refId": "D" } ], @@ -1139,6 +1108,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1152,6 +1122,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, @@ -1175,7 +1146,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1319,7 +1291,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum by (status) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | __error__=\"\" [$__interval]))", + "expr": "sum by (status) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | __error__=\"\" [$__interval]))", "legendFormat": "HTTP Status {{status}}", "refId": "A" } @@ -1356,6 +1328,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1369,6 +1342,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, @@ -1390,7 +1364,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1459,17 +1434,16 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum by (host) (sum_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | status=200 | unwrap body_bytes_sent | __error__=\"\" [$__interval]))", + "expr": "sum by (host) (sum_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | status=200 | unwrap body_bytes_sent | __error__=\"\" [$__interval]))", "legendFormat": "Bytes sent", "refId": "A" } ], "title": "Bytes Sent", - "transformations": [ ], "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "datasource": { "uid": "$datasource" }, @@ -1480,487 +1454,7 @@ "y": 30 }, "id": 28, - "panels": [ - { - "datasource": { - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "cellOptions": { - "type": "auto" - }, - "filterable": false - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Requests" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "color", - "value": { - "mode": "continuous-BlPu" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Requests" - }, - "properties": [ - { - "id": "custom.width", - "value": 300 - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 9, - "x": 0, - "y": 31 - }, - "id": 6, - "maxDataPoints": 1, - "options": { - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Requests" - } - ] - }, - "pluginVersion": "8.2.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "topk(10, sum by (http_referer) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | http_referer != \"\" and http_referer !~ \".*?$host.*?\" and http_referer !~ \".*?\\\\*\\\\*\\\\*.*?\" | __error__=\"\" [15m])))", - "instant": true, - "legendFormat": "{{http_referer}}", - "range": false, - "refId": "A" - } - ], - "timeFrom": "15m", - "title": "Top 10 HTTP Referers", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true - }, - "indexByName": { }, - "renameByName": { - "Field": "Referer", - "Total": "Requests", - "Value #A": "Requests", - "http_referer": "HTTP Referrer" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "cellOptions": { - "type": "auto" - }, - "filterable": false - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Requests" - }, - "properties": [ - { - "id": "custom.width", - "value": 300 - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "color", - "value": { - "mode": "continuous-BlPu" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 15, - "x": 9, - "y": 31 - }, - "id": 7, - "maxDataPoints": 1, - "options": { - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Requests" - } - ] - }, - "pluginVersion": "8.2.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "topk(10, sum by (http_user_agent) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | __error__=\"\" [15m])))", - "instant": true, - "legendFormat": "{{http_user_agent}}", - "range": false, - "refId": "A" - } - ], - "timeFrom": "15m", - "title": "Top 10 User Agents", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Field": false, - "Time": true - }, - "indexByName": { }, - "renameByName": { - "Field": "Agent", - "Total": "Requests", - "Value #A": "Requests", - "http_user_agent": "User agent" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "cellOptions": { - "type": "auto" - }, - "filterable": false - }, - "mappings": [ - { - "options": { - "NL": { - "index": 1, - "text": "🇳🇱" - }, - "US": { - "index": 0, - "text": "🇺🇸" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Requests" - }, - "properties": [ - { - "id": "custom.width", - "value": 300 - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "color", - "value": { - "mode": "continuous-BlPu" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Country" - }, - "properties": [ - { - "id": "custom.width", - "value": 74 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 9, - "x": 0, - "y": 37 - }, - "id": 3, - "maxDataPoints": 1, - "options": { - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Requests" - } - ] - }, - "pluginVersion": "8.2.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "topk(10, sum by (remote_addr, geoip_country_code) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} | json | __error__=\"\" [15m])))", - "instant": true, - "legendFormat": "{{remote_addr}}", - "range": false, - "refId": "A" - } - ], - "timeFrom": "15m", - "title": "Top 10 visitor IPs", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Field": false, - "Time": true - }, - "indexByName": { - "Time": 0, - "Value #A": 3, - "geoip_country_code": 2, - "remote_addr": 1 - }, - "renameByName": { - "Field": "IP Address", - "Total": "Requests", - "Value #A": "Requests", - "geoip_country_code": "Country", - "remote_addr": "IP Address " - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "cellOptions": { - "type": "auto" - }, - "filterable": false - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Requests" - }, - "properties": [ - { - "id": "custom.width", - "value": 300 - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "color", - "value": { - "mode": "continuous-BlPu" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 15, - "x": 9, - "y": 37 - }, - "id": 12, - "maxDataPoints": 1, - "options": { - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Requests" - } - ] - }, - "pluginVersion": "8.2.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "topk(10, sum by (request_uri) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=\"$cluster\"} !~ `\\.ico|\\.svg|\\.css|\\.png|\\.txt|\\.js|\\.xml` | json | status = 200 and request_uri != \"/\" | __error__=\"\" [15m])))", - "instant": true, - "legendFormat": "{{request_uri}}", - "range": false, - "refId": "A" - } - ], - "timeFrom": "15m", - "title": "Top 10 Requested Pages", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true - }, - "indexByName": { }, - "renameByName": { - "Field": "Page", - "Time": "", - "Total": "", - "Value #A": "Requests", - "request_uri": "Path" - } - } - } - ], - "type": "table" - } - ], + "panels": [ ], "targets": [ { "datasource": { @@ -1971,14 +1465,539 @@ ], "title": "Acquisition and Behaviour", "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Requests" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-BlPu" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Requests" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 0, + "y": 31 + }, + "id": 6, + "maxDataPoints": 1, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Requests" + } + ] + }, + "pluginVersion": "11.1.0-71799", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "topk(10, sum by (http_referer) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | http_referer != \"\" and http_referer !~ \".*?$host.*?\" and http_referer !~ \".*?\\\\*\\\\*\\\\*.*?\" | __error__=\"\" [$__interval])))", + "instant": true, + "legendFormat": "{{http_referer}}", + "range": false, + "refId": "A" + } + ], + "timeFrom": "15m", + "title": "Top 10 HTTP Referers", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { }, + "renameByName": { + "Field": "Referer", + "Total": "Requests", + "Value #A": "Requests", + "http_referer": "HTTP Referrer" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Requests" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-BlPu" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 15, + "x": 9, + "y": 31 + }, + "id": 7, + "maxDataPoints": 1, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Requests" + } + ] + }, + "pluginVersion": "11.1.0-71799", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "topk(10, sum by (http_user_agent) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | __error__=\"\" [$__interval])))", + "instant": true, + "legendFormat": "{{http_user_agent}}", + "range": false, + "refId": "A" + } + ], + "timeFrom": "15m", + "title": "Top 10 User Agents", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Field": false, + "Time": true + }, + "indexByName": { }, + "renameByName": { + "Field": "Agent", + "Total": "Requests", + "Value #A": "Requests", + "http_user_agent": "User agent" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [ + { + "options": { + "NL": { + "index": 1, + "text": "🇳🇱" + }, + "US": { + "index": 0, + "text": "🇺🇸" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Requests" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-BlPu" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Country" + }, + "properties": [ + { + "id": "custom.width", + "value": 74 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 0, + "y": 37 + }, + "id": 3, + "maxDataPoints": 1, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Requests" + } + ] + }, + "pluginVersion": "11.1.0-71799", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "topk(10, sum by (remote_addr, geoip_country_code) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} | json | __error__=\"\" [$__interval])))", + "instant": true, + "legendFormat": "{{remote_addr}}", + "range": false, + "refId": "A" + } + ], + "timeFrom": "15m", + "title": "Top 10 visitor IPs", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Field": false, + "Time": true + }, + "indexByName": { + "Time": 0, + "Value #A": 3, + "geoip_country_code": 2, + "remote_addr": 1 + }, + "renameByName": { + "Field": "IP Address", + "Total": "Requests", + "Value #A": "Requests", + "geoip_country_code": "Country", + "remote_addr": "IP Address " + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Requests" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-BlPu" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 15, + "x": 9, + "y": 37 + }, + "id": 12, + "maxDataPoints": 1, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Requests" + } + ] + }, + "pluginVersion": "11.1.0-71799", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "topk(10, sum by (request_uri) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"} !~ `\\.ico|\\.svg|\\.css|\\.png|\\.txt|\\.js|\\.xml` | json | status = 200 and request_uri != \"/\" | __error__=\"\" [$__interval])))", + "instant": true, + "legendFormat": "{{request_uri}}", + "range": false, + "refId": "A" + } + ], + "timeFrom": "15m", + "title": "Top 10 Requested Pages", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { }, + "renameByName": { + "Field": "Page", + "Time": "", + "Total": "", + "Value #A": "Requests", + "request_uri": "Path" + } + } + } + ], + "type": "table" } ], - "refresh": "", - "schemaVersion": 38, - "style": "dark", + "refresh": "30s", + "schemaVersion": 39, "tags": [ - "loki", - "ds-example" + "nginx-integration" ], "templating": { "list": [ @@ -1993,7 +2012,7 @@ "query": "loki", "queryValue": "", "refresh": 1, - "regex": "", + "regex": "(?!grafanacloud.+usage-insights|grafanacloud.+alert-state-history).+", "skipUrlSync": false, "type": "datasource" }, @@ -2016,7 +2035,39 @@ "useTags": false }, { - "current": { }, + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "$datasource" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Cluster name", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values({},cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "job", + "value": "job" + }, "datasource": { "type": "loki", "uid": "$datasource" @@ -2040,7 +2091,15 @@ }, { "allValue": ".+", - "current": { }, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, "datasource": { "type": "loki", "uid": "$datasource" @@ -2064,19 +2123,27 @@ }, { "allValue": ".*", - "current": { }, + "current": { + "selected": true, + "text": [ + "integrations/nginx" + ], + "value": [ + "integrations/nginx" + ] + }, "datasource": { "type": "loki", "uid": "$datasource" }, - "definition": "label_values({$label_name=~\"$label_value\", cluster=\"$cluster\"}, job)", + "definition": "label_values({$label_name=~\"$label_value\", cluster=~\"$cluster\"}, job)", "hide": 0, "includeAll": true, "label": "Job", "multi": true, "name": "job", "options": [ ], - "query": "label_values({$label_name=~\"$label_value\", cluster=\"$cluster\"}, job)", + "query": "label_values({$label_name=~\"$label_value\", cluster=~\"$cluster\"}, job)", "refresh": 2, "regex": "", "skipUrlSync": false, @@ -2088,19 +2155,27 @@ }, { "allValue": ".*", - "current": { }, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, "datasource": { "type": "loki", "uid": "$datasource" }, - "definition": "label_values({$label_name=~\"$label_value\", cluster=\"$cluster\"}, instance)", + "definition": "label_values({$label_name=~\"$label_value\", cluster=~\"$cluster\"}, instance)", "hide": 0, "includeAll": true, "label": "Instance", "multi": true, "name": "instance", "options": [ ], - "query": "label_values({$label_name=~\"$label_value\", cluster=\"$cluster\"}, instance)", + "query": "label_values({$label_name=~\"$label_value\", cluster=~\"$cluster\"}, instance)", "refresh": 2, "regex": "", "skipUrlSync": false, @@ -2130,8 +2205,8 @@ ] }, "timezone": "", - "title": "Loki NGINX Service Mesh - JSON version", + "title": "NGINX Logs", "uid": "T512JVH7z", - "version": 18, + "version": 4, "weekStart": "" } diff --git a/assets/nginx/dashboards/nginx-metrics.json b/assets/nginx/dashboards/nginx-metrics.json new file mode 100644 index 0000000..cfe5741 --- /dev/null +++ b/assets/nginx/dashboards/nginx-metrics.json @@ -0,0 +1,551 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [ ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ ], + "title": "Status", + "type": "row" + }, + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "This panel indicates the current status of the NGINX server for the selected instance, showing whether it's up or down.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Down" + }, + "1": { + "color": "green", + "index": 0, + "text": "Up" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.2+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "nginx_up{instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "NGINX Status for $instance", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 3, + "panels": [ ], + "title": "Metrics", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "This panel tracks the number of processed connections for the selected instance, displaying both active and handled connections to monitor server activity.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Connections (rate)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(nginx_connections_accepted{instance=~\"$instance\"}[5m])", + "instant": false, + "legendFormat": "{{instance}} accepted", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(nginx_connections_handled{instance=~\"$instance\"}[5m])", + "hide": false, + "instant": false, + "legendFormat": "{{instance}} handled", + "range": true, + "refId": "B" + } + ], + "title": "Processed connections", + "type": "timeseries" + }, + { + "datasource": { + "name": "${datasource}", + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "This panel shows NGINX active connections for the selected instance, categorized as writing, waiting, reading, and total active.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Connections", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 29, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "nginx_connections_active{instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}} active", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "nginx_connections_reading{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "{{instance}} reading", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "nginx_connections_waiting{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "{{instance}} waiting", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "nginx_connections_writing{instance=~\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "{{instance}} writing", + "range": true, + "refId": "D" + } + ], + "title": "Active Connections", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "This panel displays the total number of requests processed by the NGINX server for the selected instance, providing insight into traffic volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 3, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(nginx_http_requests_total{instance=~\"$instance\"}[5m])", + "instant": false, + "legendFormat": "{{instance}} total requests", + "range": true, + "refId": "A" + } + ], + "title": "Total requests", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [ + "NGINX" + ], + "templating": { + "list": [ + { + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(nginx_up,instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [ ], + "query": { + "qryType": 1, + "query": "label_values(nginx_up,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { }, + "timezone": "browser", + "title": "NGINX Overview", + "uid": "MsjffzSZz", + "version": 4, + "weekStart": "" +} diff --git a/assets/node-exporter/alerts.yaml b/assets/node-exporter/alerts.yaml index 75df6ae..ed790da 100644 --- a/assets/node-exporter/alerts.yaml +++ b/assets/node-exporter/alerts.yaml @@ -288,7 +288,7 @@ groups: - alert: NodeDiskIOSaturation annotations: description: | - Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}. + Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 30 minutes, is currently at {{ printf "%.2f" $value }}. This symptom might indicate disk saturation. summary: Disk IO queue is high. expr: | @@ -306,3 +306,13 @@ groups: for: 5m labels: severity: warning + - alert: NodeBondingDegraded + annotations: + description: Bonding interface {{ $labels.master }} on {{ $labels.instance }} + is in degraded state due to one or more slave failures. + summary: Bonding interface is degraded + expr: | + (node_bonding_slaves - node_bonding_active) != 0 + for: 5m + labels: + severity: warning diff --git a/assets/node-exporter/dashboards/node-cluster-rsrc-use.json b/assets/node-exporter/dashboards/node-cluster-rsrc-use.json index 6ba92b2..b40d090 100644 --- a/assets/node-exporter/dashboards/node-cluster-rsrc-use.json +++ b/assets/node-exporter/dashboards/node-cluster-rsrc-use.json @@ -918,5 +918,6 @@ }, "timezone": "utc", "title": "Node Exporter / USE Method / Cluster", + "uid": "3e97d1d02672cdd0861f4c97c64f89b2", "version": 0 } diff --git a/assets/node-exporter/dashboards/node-rsrc-use.json b/assets/node-exporter/dashboards/node-rsrc-use.json index 7bf869d..1f8c1c8 100644 --- a/assets/node-exporter/dashboards/node-rsrc-use.json +++ b/assets/node-exporter/dashboards/node-rsrc-use.json @@ -938,5 +938,6 @@ }, "timezone": "utc", "title": "Node Exporter / USE Method / Node", + "uid": "fac67cfbe174d3ef53eb473d73d9212f", "version": 0 } diff --git a/assets/node-exporter/dashboards/nodes-aix.json b/assets/node-exporter/dashboards/nodes-aix.json new file mode 100644 index 0000000..72b9fad --- /dev/null +++ b/assets/node-exporter/dashboards/nodes-aix.json @@ -0,0 +1,948 @@ +{ + "__inputs": [ ], + "__requires": [ ], + "annotations": { + "list": [ ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node\", mode=~\"idle|iowait|steal\", instance=\"$instance\", cluster=\"$cluster\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node\", mode=\"idle\", instance=\"$instance\", cluster=\"$cluster\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{cpu}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_total_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Physical Memory", + "refId": "A" + }, + { + "expr": "(\n node_memory_total_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} -\n node_memory_available_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory Used", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "100 -\n(\n avg(node_memory_available_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}) /\n avg(node_memory_total_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"})\n * 100\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} io time", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.80000000000000004 + }, + { + "color": "red", + "value": 0.90000000000000002 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { }, + "indexByName": { }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(node_uname_info{job=\"node\", sysname!=\"Darwin\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ ], + "query": "label_values(node_uname_info{job=\"node\", cluster=\"$cluster\", sysname!=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Node Exporter / AIX", + "uid": "7e0a61e486f727d763fb1d86fdd629c2", + "version": 0 +} diff --git a/assets/node-exporter/dashboards/nodes-darwin.json b/assets/node-exporter/dashboards/nodes-darwin.json index 811cb15..ecdce38 100644 --- a/assets/node-exporter/dashboards/nodes-darwin.json +++ b/assets/node-exporter/dashboards/nodes-darwin.json @@ -54,7 +54,7 @@ "steppedLine": false, "targets": [ { - "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node\", mode=~\"idle|iowait|steal\", instance=\"$instance\", cluster=\"$cluster\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node\", mode=\"idle\", instance=\"$instance\", cluster=\"$cluster\"})\n)\n", "format": "time_series", "intervalFactor": 5, "legendFormat": "{{cpu}}", @@ -135,28 +135,28 @@ "steppedLine": false, "targets": [ { - "expr": "node_load1{job=\"node\", instance=\"$instance\"}", + "expr": "node_load1{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "1m load average", "refId": "A" }, { - "expr": "node_load5{job=\"node\", instance=\"$instance\"}", + "expr": "node_load5{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "5m load average", "refId": "B" }, { - "expr": "node_load15{job=\"node\", instance=\"$instance\"}", + "expr": "node_load15{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "15m load average", "refId": "C" }, { - "expr": "count(node_cpu_seconds_total{job=\"node\", instance=\"$instance\", mode=\"idle\"})", + "expr": "count(node_cpu_seconds_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", mode=\"idle\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "logical cores", @@ -250,35 +250,35 @@ "steppedLine": false, "targets": [ { - "expr": "node_memory_total_bytes{job=\"node\", instance=\"$instance\"}", + "expr": "node_memory_total_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Physical Memory", "refId": "A" }, { - "expr": "(\n node_memory_internal_bytes{job=\"node\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node\", instance=\"$instance\"} +\n node_memory_wired_bytes{job=\"node\", instance=\"$instance\"} +\n node_memory_compressed_bytes{job=\"node\", instance=\"$instance\"}\n)\n", + "expr": "(\n node_memory_internal_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} -\n node_memory_purgeable_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} +\n node_memory_wired_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} +\n node_memory_compressed_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "Memory Used", "refId": "B" }, { - "expr": "(\n node_memory_internal_bytes{job=\"node\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node\", instance=\"$instance\"}\n)\n", + "expr": "(\n node_memory_internal_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} -\n node_memory_purgeable_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "App Memory", "refId": "C" }, { - "expr": "node_memory_wired_bytes{job=\"node\", instance=\"$instance\"}", + "expr": "node_memory_wired_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Wired Memory", "refId": "D" }, { - "expr": "node_memory_compressed_bytes{job=\"node\", instance=\"$instance\"}", + "expr": "node_memory_compressed_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Compressed", @@ -351,7 +351,7 @@ "span": 3, "targets": [ { - "expr": "(\n (\n avg(node_memory_internal_bytes{job=\"node\", instance=\"$instance\"}) -\n avg(node_memory_purgeable_bytes{job=\"node\", instance=\"$instance\"}) +\n avg(node_memory_wired_bytes{job=\"node\", instance=\"$instance\"}) +\n avg(node_memory_compressed_bytes{job=\"node\", instance=\"$instance\"})\n ) /\n avg(node_memory_total_bytes{job=\"node\", instance=\"$instance\"})\n)\n*\n100\n", + "expr": "(\n (\n avg(node_memory_internal_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}) -\n avg(node_memory_purgeable_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}) +\n avg(node_memory_wired_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}) +\n avg(node_memory_compressed_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"})\n ) /\n avg(node_memory_total_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"})\n)\n*\n100\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "" @@ -421,21 +421,21 @@ "steppedLine": false, "targets": [ { - "expr": "rate(node_disk_read_bytes_total{job=\"node\", instance=\"$instance\", device!=\"\"}[$__rate_interval])", + "expr": "rate(node_disk_read_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} read", "refId": "A" }, { - "expr": "rate(node_disk_written_bytes_total{job=\"node\", instance=\"$instance\", device!=\"\"}[$__rate_interval])", + "expr": "rate(node_disk_written_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} written", "refId": "B" }, { - "expr": "rate(node_disk_io_time_seconds_total{job=\"node\", instance=\"$instance\", device!=\"\"}[$__rate_interval])", + "expr": "rate(node_disk_io_time_seconds_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} io time", @@ -581,14 +581,14 @@ "span": 6, "targets": [ { - "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", fstype!=\"\", mountpoint!=\"\"})\n", "format": "table", "instant": true, "intervalFactor": 2, "legendFormat": "" }, { - "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", fstype!=\"\", mountpoint!=\"\"})\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -735,7 +735,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(node_network_receive_bytes_total{job=\"node\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "expr": "rate(node_network_receive_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"lo\"}[$__rate_interval]) * 8", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}}", @@ -817,7 +817,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(node_network_transmit_bytes_total{job=\"node\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "expr": "rate(node_network_transmit_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"lo\"}[$__rate_interval]) * 8", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}}", @@ -891,6 +891,26 @@ "regex": "", "type": "datasource" }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(node_uname_info{job=\"node\", sysname=\"Darwin\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": null, "current": { }, @@ -901,7 +921,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(node_uname_info{job=\"node\", sysname=\"Darwin\"}, instance)", + "query": "label_values(node_uname_info{job=\"node\", cluster=\"$cluster\", sysname=\"Darwin\"}, instance)", "refresh": 2, "regex": "", "sort": 0, @@ -944,5 +964,6 @@ }, "timezone": "utc", "title": "Node Exporter / MacOS", + "uid": "629701ea43bf69291922ea45f4a87d37", "version": 0 } diff --git a/assets/node-exporter/dashboards/nodes.json b/assets/node-exporter/dashboards/nodes.json index b3e5a99..f19045f 100644 --- a/assets/node-exporter/dashboards/nodes.json +++ b/assets/node-exporter/dashboards/nodes.json @@ -54,7 +54,7 @@ "steppedLine": false, "targets": [ { - "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node\", mode=~\"idle|iowait|steal\", instance=\"$instance\", cluster=\"$cluster\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node\", mode=\"idle\", instance=\"$instance\", cluster=\"$cluster\"})\n)\n", "format": "time_series", "intervalFactor": 5, "legendFormat": "{{cpu}}", @@ -135,28 +135,28 @@ "steppedLine": false, "targets": [ { - "expr": "node_load1{job=\"node\", instance=\"$instance\"}", + "expr": "node_load1{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "1m load average", "refId": "A" }, { - "expr": "node_load5{job=\"node\", instance=\"$instance\"}", + "expr": "node_load5{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "5m load average", "refId": "B" }, { - "expr": "node_load15{job=\"node\", instance=\"$instance\"}", + "expr": "node_load15{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "15m load average", "refId": "C" }, { - "expr": "count(node_cpu_seconds_total{job=\"node\", instance=\"$instance\", mode=\"idle\"})", + "expr": "count(node_cpu_seconds_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", mode=\"idle\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "logical cores", @@ -250,28 +250,28 @@ "steppedLine": false, "targets": [ { - "expr": "(\n node_memory_MemTotal_bytes{job=\"node\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node\", instance=\"$instance\"}\n)\n", + "expr": "(\n node_memory_MemTotal_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n-\n node_memory_MemFree_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n-\n node_memory_Buffers_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n-\n node_memory_Cached_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory used", "refId": "A" }, { - "expr": "node_memory_Buffers_bytes{job=\"node\", instance=\"$instance\"}", + "expr": "node_memory_Buffers_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory buffers", "refId": "B" }, { - "expr": "node_memory_Cached_bytes{job=\"node\", instance=\"$instance\"}", + "expr": "node_memory_Cached_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory cached", "refId": "C" }, { - "expr": "node_memory_MemFree_bytes{job=\"node\", instance=\"$instance\"}", + "expr": "node_memory_MemFree_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory free", @@ -344,7 +344,7 @@ "span": 3, "targets": [ { - "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node\", instance=\"$instance\"}) /\n avg(node_memory_MemTotal_bytes{job=\"node\", instance=\"$instance\"})\n* 100\n)\n", + "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}) /\n avg(node_memory_MemTotal_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"})\n* 100\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "" @@ -414,21 +414,21 @@ "steppedLine": false, "targets": [ { - "expr": "rate(node_disk_read_bytes_total{job=\"node\", instance=\"$instance\", device!=\"\"}[$__rate_interval])", + "expr": "rate(node_disk_read_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} read", "refId": "A" }, { - "expr": "rate(node_disk_written_bytes_total{job=\"node\", instance=\"$instance\", device!=\"\"}[$__rate_interval])", + "expr": "rate(node_disk_written_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} written", "refId": "B" }, { - "expr": "rate(node_disk_io_time_seconds_total{job=\"node\", instance=\"$instance\", device!=\"\"}[$__rate_interval])", + "expr": "rate(node_disk_io_time_seconds_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} io time", @@ -574,14 +574,14 @@ "span": 6, "targets": [ { - "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", fstype!=\"\", mountpoint!=\"\"})\n", "format": "table", "instant": true, "intervalFactor": 2, "legendFormat": "" }, { - "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", fstype!=\"\", mountpoint!=\"\"})\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -728,7 +728,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(node_network_receive_bytes_total{job=\"node\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "expr": "rate(node_network_receive_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"lo\"}[$__rate_interval]) * 8", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}}", @@ -810,7 +810,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(node_network_transmit_bytes_total{job=\"node\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "expr": "rate(node_network_transmit_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"lo\"}[$__rate_interval]) * 8", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}}", @@ -884,6 +884,26 @@ "regex": "", "type": "datasource" }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(node_uname_info{job=\"node\", sysname!=\"Darwin\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": null, "current": { }, @@ -894,7 +914,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(node_uname_info{job=\"node\", sysname!=\"Darwin\"}, instance)", + "query": "label_values(node_uname_info{job=\"node\", cluster=\"$cluster\", sysname!=\"Darwin\"}, instance)", "refresh": 2, "regex": "", "sort": 0, @@ -937,5 +957,6 @@ }, "timezone": "utc", "title": "Node Exporter / Nodes", + "uid": "7d57716318ee0dddbac5a7f451fb7753", "version": 0 } diff --git a/assets/openldap/dashboards/logs b/assets/openldap/dashboards/logs index 14dfd04..7c6b390 100644 --- a/assets/openldap/dashboards/logs +++ b/assets/openldap/dashboards/logs @@ -138,17 +138,19 @@ }, "gridPos": { "h": 6, - "w": 24 + "w": 24, + "x": 0, + "y": 0 }, "id": 1, - "interval": "30s", + "maxDataPoints": 100, "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -178,7 +180,9 @@ }, "gridPos": { "h": 18, - "w": 24 + "w": 24, + "x": 0, + "y": 18 }, "id": 2, "options": { @@ -188,7 +192,7 @@ "showTime": false, "wrapLogMessage": false }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -203,7 +207,7 @@ } ], "refresh": "1m", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "openldap" ], diff --git a/assets/openldap/dashboards/overview b/assets/openldap/dashboards/overview index 0731f6c..904aa41 100644 --- a/assets/openldap/dashboards/overview +++ b/assets/openldap/dashboards/overview @@ -52,7 +52,7 @@ }, "textMode": "value" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -106,7 +106,7 @@ }, "textMode": "value" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -187,7 +187,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -250,7 +250,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -320,7 +320,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -383,7 +383,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -446,7 +446,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -509,7 +509,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -578,7 +578,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -683,7 +683,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -791,7 +791,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -878,7 +878,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { diff --git a/assets/opensearch/alerts.yaml b/assets/opensearch/alerts.yaml index 271d085..8030045 100644 --- a/assets/opensearch/alerts.yaml +++ b/assets/opensearch/alerts.yaml @@ -6,7 +6,7 @@ groups: description: '{{$labels.cluster}} health status is yellow over the last 5 minutes' summary: At least one of the clusters is reporting a yellow status. expr: | - opensearch_cluster_status{cluster!=""} == 1 + opensearch_cluster_status{opensearch_cluster!=""} == 1 for: 5m labels: severity: warning @@ -15,7 +15,7 @@ groups: description: '{{$labels.cluster}} health status is red over the last 5 minutes' summary: At least one of the clusters is reporting a red status. expr: | - opensearch_cluster_status{cluster!=""} == 2 + opensearch_cluster_status{opensearch_cluster!=""} == 2 for: 5m labels: severity: critical @@ -25,7 +25,7 @@ groups: {{$labels.cluster}} has had {{ printf "%.0f" $value }} shard reallocation over the last 1m which is above the threshold of 0. summary: A node has gone offline or has been disconnected triggering shard reallocation. expr: | - sum without(type) (opensearch_cluster_shards_number{cluster!="", type="relocating"}) > 0 + sum without(type) (opensearch_cluster_shards_number{opensearch_cluster!="", type="relocating"}) > 0 for: 1m labels: severity: warning @@ -35,17 +35,17 @@ groups: {{$labels.cluster}} has had {{ printf "%.0f" $value }} shard unassigned over the last 5m which is above the threshold of 0. summary: There are shards that have been detected as unassigned. expr: | - sum without(type) (opensearch_cluster_shards_number{cluster!="", type="unassigned"}) > 0 + sum without(type) (opensearch_cluster_shards_number{opensearch_cluster!="", type="unassigned"}) > 0 for: 5m labels: severity: warning - - alert: OpenSearchModerateNodeDiskUsage + - alert: OpenSearchHighNodeDiskUsage annotations: description: | {{$labels.node}} has had {{ printf "%.0f" $value }} disk usage over the last 5m which is above the threshold of 60. summary: The node disk usage has exceeded the warning threshold. expr: | - 100 * sum without(nodeid, path, mount, type) ((opensearch_fs_path_total_bytes{cluster!=""} - opensearch_fs_path_free_bytes{cluster!=""}) / opensearch_fs_path_total_bytes{cluster!=""}) > 60 + 100 * sum without(nodeid, path, mount, type) ((opensearch_fs_path_total_bytes{opensearch_cluster!=""} - opensearch_fs_path_free_bytes{opensearch_cluster!=""}) / opensearch_fs_path_total_bytes{opensearch_cluster!=""}) > 60 for: 5m labels: severity: warning @@ -55,17 +55,17 @@ groups: {{$labels.node}} has had {{ printf "%.0f" $value }}% disk usage over the last 5m which is above the threshold of 80. summary: The node disk usage has exceeded the critical threshold. expr: | - 100 * sum without(nodeid, path, mount, type) ((opensearch_fs_path_total_bytes{cluster!=""} - opensearch_fs_path_free_bytes) / opensearch_fs_path_total_bytes{cluster!=""}) > 80 + 100 * sum without(nodeid, path, mount, type) ((opensearch_fs_path_total_bytes{opensearch_cluster!=""} - opensearch_fs_path_free_bytes) / opensearch_fs_path_total_bytes{opensearch_cluster!=""}) > 80 for: 5m labels: severity: critical - - alert: OpenSearchModerateNodeCpuUsage + - alert: OpenSearchHighNodeCpuUsage annotations: description: | {{$labels.node}} has had {{ printf "%.0f" $value }}% CPU usage over the last 5m which is above the threshold of 70. summary: The node CPU usage has exceeded the warning threshold. expr: | - sum without(nodeid) (opensearch_os_cpu_percent{cluster!=""}) > 70 + sum without(nodeid) (opensearch_os_cpu_percent{opensearch_cluster!=""}) > 70 for: 5m labels: severity: warning @@ -75,17 +75,17 @@ groups: {{$labels.node}} has had {{ printf "%.0f" $value }}% CPU usage over the last 5m which is above the threshold of 85. summary: The node CPU usage has exceeded the critical threshold. expr: | - sum without(nodeid) (opensearch_os_cpu_percent{cluster!=""}) > 85 + sum without(nodeid) (opensearch_os_cpu_percent{opensearch_cluster!=""}) > 85 for: 5m labels: severity: critical - - alert: OpenSearchModerateNodeMemoryUsage + - alert: OpenSearchHighNodeMemoryUsage annotations: description: | {{$labels.node}} has had {{ printf "%.0f" $value }}% memory usage over the last 5m which is above the threshold of 70. summary: The node memory usage has exceeded the warning threshold. expr: | - sum without(nodeid) (opensearch_os_mem_used_percent{cluster!=""}) > 70 + sum without(nodeid) (opensearch_os_mem_used_percent{opensearch_cluster!=""}) > 70 for: 5m labels: severity: warning @@ -95,7 +95,7 @@ groups: {{$labels.node}} has had {{ printf "%.0f" $value }}% memory usage over the last 5m which is above the threshold of 85. summary: The node memory usage has exceeded the critical threshold. expr: | - sum without(nodeid) (opensearch_os_mem_used_percent{cluster!=""}) > 85 + sum without(nodeid) (opensearch_os_mem_used_percent{opensearch_cluster!=""}) > 85 for: 5m labels: severity: critical @@ -105,7 +105,7 @@ groups: {{$labels.index}} has had {{ printf "%.0f" $value }}s of request latency over the last 5m which is above the threshold of 0.5. summary: The request latency has exceeded the warning threshold. expr: | - sum without(context) ((increase(opensearch_index_search_fetch_time_seconds{cluster!="", context="total"}[5m])+increase(opensearch_index_search_query_time_seconds{context="total"}[5m])+increase(opensearch_index_search_scroll_time_seconds{context="total"}[5m])) / clamp_min(increase(opensearch_index_search_fetch_count{context="total"}[5m])+increase(opensearch_index_search_query_count{context="total"}[5m])+increase(opensearch_index_search_scroll_count{context="total"}[5m]), 1)) > 0.5 + sum without(context) ((increase(opensearch_index_search_fetch_time_seconds{opensearch_cluster!="", context="total"}[5m])+increase(opensearch_index_search_query_time_seconds{context="total"}[5m])+increase(opensearch_index_search_scroll_time_seconds{context="total"}[5m])) / clamp_min(increase(opensearch_index_search_fetch_count{context="total"}[5m])+increase(opensearch_index_search_query_count{context="total"}[5m])+increase(opensearch_index_search_scroll_count{context="total"}[5m]), 1)) > 0.5 for: 5m labels: severity: warning @@ -115,7 +115,7 @@ groups: {{$labels.index}} has had {{ printf "%.0f" $value }}s of index latency over the last 5m which is above the threshold of 0.5. summary: The index latency has exceeded the warning threshold. expr: | - sum without(context) (increase(opensearch_index_indexing_index_time_seconds{cluster!="", context="total"}[5m]) / clamp_min(increase(opensearch_index_indexing_index_count{context="total"}[5m]), 1)) > 0.5 + sum without(context) (increase(opensearch_index_indexing_index_time_seconds{opensearch_cluster!="", context="total"}[5m]) / clamp_min(increase(opensearch_index_indexing_index_count{context="total"}[5m]), 1)) > 0.5 for: 5m labels: severity: warning diff --git a/assets/opensearch/dashboards/node-overview.json b/assets/opensearch/dashboards/node-overview.json index 6643a3c..75ce567 100644 --- a/assets/opensearch/dashboards/node-overview.json +++ b/assets/opensearch/dashboards/node-overview.json @@ -71,41 +71,41 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\", role=\"data\"}[1m]) == 1\n) * 2\n", + "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\", role=\"data\"}[1m]) == 1\n) * 2\n", "legendFormat": "{{node}}" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\", role=\"master\"}[1m]) == 1\n) * 3\n", + "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\", role=\"master\"}[1m]) == 1\n) * 3\n", "legendFormat": "{{node}}" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\", role=\"ingest\"}[1m]) == 1\n) * 4\n", + "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\", role=\"ingest\"}[1m]) == 1\n) * 4\n", "legendFormat": "{{node}}" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\", role=\"cluster_manager\"}[1m]) == 1\n) * 5\n", + "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\", role=\"cluster_manager\"}[1m]) == 1\n) * 5\n", "legendFormat": "{{node}}" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\", role=\"remote_cluster_client\"}[1m]) == 1\n) * 6\n", + "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\", role=\"remote_cluster_client\"}[1m]) == 1\n) * 6\n", "legendFormat": "{{node}}" } ], @@ -115,7 +115,7 @@ { "collapsed": false, "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "gridPos": { "h": 1, @@ -131,7 +131,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "CPU usage percentage of the node's Operating System.", "fieldConfig": { @@ -169,14 +169,14 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "opensearch_os_cpu_percent{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"}", + "expr": "opensearch_os_cpu_percent{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"}", "legendFormat": "{{node}}" } ], @@ -186,7 +186,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Memory usage percentage of the node for the Operating System and OpenSearch", "fieldConfig": { @@ -224,14 +224,14 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "opensearch_os_mem_used_percent{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"}", + "expr": "opensearch_os_mem_used_percent{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"}", "legendFormat": "{{node}}" } ], @@ -241,7 +241,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Node file system read and write data.", "fieldConfig": { @@ -296,22 +296,22 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by(job,cluster,node) (rate(opensearch_fs_io_total_read_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"}[$__rate_interval]))", + "expr": "sum by(job,opensearch_cluster,node) (rate(opensearch_fs_io_total_read_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{node}} - read" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by(job,cluster,node) (rate(opensearch_fs_io_total_write_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"}[$__rate_interval]))", + "expr": "sum by(job,opensearch_cluster,node) (rate(opensearch_fs_io_total_write_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"}[$__rate_interval]))", "legendFormat": "{{node}} - write" } ], @@ -321,7 +321,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Number of open connections for the selected node.", "fieldConfig": { @@ -354,14 +354,14 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by (job,cluster,node) (opensearch_transport_server_open_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"})", + "expr": "sum by (job,opensearch_cluster,node) (opensearch_transport_server_open_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"})", "legendFormat": "{{node}}" } ], @@ -371,7 +371,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Disk usage percentage of the selected node.", "fieldConfig": { @@ -409,14 +409,14 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "100 - (100 * opensearch_fs_path_free_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"} / clamp_min(opensearch_fs_path_total_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"}, 1))", + "expr": "100 - (100 * opensearch_fs_path_free_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"} / clamp_min(opensearch_fs_path_total_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"}, 1))", "legendFormat": "{{node}}" } ], @@ -426,7 +426,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Percentage of swap space used by OpenSearch and the Operating System on the selected node.", "fieldConfig": { @@ -464,14 +464,14 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "100 * opensearch_os_swap_used_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"} / clamp_min((opensearch_os_swap_used_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"} + opensearch_os_swap_free_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"}), 1)", + "expr": "100 * opensearch_os_swap_used_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"} / clamp_min((opensearch_os_swap_used_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"} + opensearch_os_swap_free_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"}), 1)", "legendFormat": "{{node}}" } ], @@ -481,7 +481,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Node network traffic sent and received.", "fieldConfig": { @@ -530,22 +530,22 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by (job,cluster,node) (rate(opensearch_transport_tx_bytes_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"}[$__rate_interval])) * 8", + "expr": "sum by (job,opensearch_cluster,node) (rate(opensearch_transport_tx_bytes_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"}[$__rate_interval])) * 8", "legendFormat": "{{node}} - sent" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by (job,cluster,node) (rate(opensearch_transport_rx_bytes_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"}[$__rate_interval])) * 8", + "expr": "sum by (job,opensearch_cluster,node) (rate(opensearch_transport_rx_bytes_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"}[$__rate_interval])) * 8", "legendFormat": "{{node}} - received" } ], @@ -554,7 +554,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Circuit breakers tripped on the selected node by type", "fieldConfig": { @@ -628,9 +628,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by(name, job,cluster,node) (increase(opensearch_circuitbreaker_tripped_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"}[$__interval:]))", + "expr": "sum by(name, job,opensearch_cluster,node) (increase(opensearch_circuitbreaker_tripped_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"}[$__interval:]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -643,7 +643,7 @@ { "collapsed": false, "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "gridPos": { "h": 1, @@ -658,7 +658,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The amount of heap memory used vs committed on the selected node.", "fieldConfig": { @@ -732,18 +732,18 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by (job,cluster,node) (opensearch_jvm_mem_heap_used_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"})", + "expr": "sum by (job,opensearch_cluster,node) (opensearch_jvm_mem_heap_used_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}} - used" }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by (job,cluster,node) (opensearch_jvm_mem_heap_committed_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"})", + "expr": "sum by (job,opensearch_cluster,node) (opensearch_jvm_mem_heap_committed_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}} - commited" @@ -754,7 +754,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The amount of non-heap memory used vs committed on the selected node.", "fieldConfig": { @@ -828,18 +828,18 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by (job,cluster,node) (opensearch_jvm_mem_nonheap_used_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"})", + "expr": "sum by (job,opensearch_cluster,node) (opensearch_jvm_mem_nonheap_used_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}} - used" }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by (job,cluster,node) (opensearch_jvm_mem_nonheap_committed_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"})", + "expr": "sum by (job,opensearch_cluster,node) (opensearch_jvm_mem_nonheap_committed_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}} - commited" @@ -850,7 +850,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The number of threads running in the JVM on the selected node.", "fieldConfig": { @@ -928,9 +928,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by (job,cluster,node) (opensearch_jvm_threads_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"})", + "expr": "sum by (job,opensearch_cluster,node) (opensearch_jvm_threads_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}" @@ -941,7 +941,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The number of buffer pools available on the selected node.", "fieldConfig": { @@ -1019,9 +1019,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by( job,cluster,node, bufferpool) (opensearch_jvm_bufferpool_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"})", + "expr": "sum by( job,opensearch_cluster,node, bufferpool) (opensearch_jvm_bufferpool_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}} - {{bufferpool}}" @@ -1032,7 +1032,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The uptime of the JVM in seconds on the selected node.", "fieldConfig": { @@ -1111,9 +1111,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by(job,cluster,node) (opensearch_jvm_uptime_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"})", + "expr": "sum by(job,opensearch_cluster,node) (opensearch_jvm_uptime_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}" @@ -1124,7 +1124,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The number of garbage collection operations on the selected node.", "fieldConfig": { @@ -1202,9 +1202,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by (job,cluster,node) (increase(opensearch_jvm_gc_collection_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"}[$__interval:]))", + "expr": "sum by (job,opensearch_cluster,node) (increase(opensearch_jvm_gc_collection_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"}[$__interval:]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1216,7 +1216,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The amount of time spent on garbage collection on the selected node.", "fieldConfig": { @@ -1290,9 +1290,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by (job,cluster,node) (increase(opensearch_jvm_gc_collection_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"}[$__interval:]))", + "expr": "sum by (job,opensearch_cluster,node) (increase(opensearch_jvm_gc_collection_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"}[$__interval:]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1304,7 +1304,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The percent used of JVM buffer pool memory.", "fieldConfig": { @@ -1382,9 +1382,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "100 * (sum by (job,cluster,node, bufferpool) (opensearch_jvm_bufferpool_used_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"})) / clamp_min((sum by (job, bufferpool, cluster) (opensearch_jvm_bufferpool_total_capacity_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"})),1)", + "expr": "100 * (sum by (job,opensearch_cluster,node, bufferpool) (opensearch_jvm_bufferpool_used_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"})) / clamp_min((sum by (job, bufferpool, cluster) (opensearch_jvm_bufferpool_total_capacity_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"})),1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}} - {{bufferpool}}" @@ -1396,7 +1396,7 @@ { "collapsed": false, "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "gridPos": { "h": 1, @@ -1411,7 +1411,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The number of threads in the thread pool for the selected node", "fieldConfig": { @@ -1488,9 +1488,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by(job,cluster,node) ((opensearch_threadpool_threads_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"}))", + "expr": "sum by(job,opensearch_cluster,node) ((opensearch_threadpool_threads_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"}))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}" @@ -1501,7 +1501,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The number of tasks in the thread pool for the selected node.", "fieldConfig": { @@ -1578,9 +1578,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by (job,cluster,node) (opensearch_threadpool_tasks_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"})", + "expr": "sum by (job,opensearch_cluster,node) (opensearch_threadpool_tasks_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}" @@ -1617,7 +1617,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",node=~\"$node\"} |~ \"\"", + "expr": "{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",node=~\"$node\"} |~ \"\"", "queryType": "range", "refId": "A" } @@ -1634,8 +1634,8 @@ "templating": { "list": [ { - "label": "Data source", - "name": "datasource", + "label": "Prometheus data source", + "name": "prometheus_datasource", "query": "prometheus", "regex": "", "type": "datasource" @@ -1644,13 +1644,13 @@ "allValue": ".+", "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "includeAll": true, "label": "Job", "multi": true, "name": "job", - "query": "label_values(opensearch_os_cpu_percent{cluster!=\"\"}, job)", + "query": "label_values(opensearch_os_cpu_percent{opensearch_cluster!=\"\"}, job)", "refresh": 2, "sort": 1, "type": "query" @@ -1659,13 +1659,13 @@ "allValue": ".+", "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "includeAll": true, - "label": "Cluster", + "label": "Opensearch_cluster", "multi": true, - "name": "cluster", - "query": "label_values(opensearch_os_cpu_percent{cluster!=\"\",job=~\"$job\"}, cluster)", + "name": "opensearch_cluster", + "query": "label_values(opensearch_os_cpu_percent{opensearch_cluster!=\"\",job=~\"$job\"}, opensearch_cluster)", "refresh": 2, "sort": 1, "type": "query" @@ -1674,16 +1674,24 @@ "allValue": ".+", "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "includeAll": true, "label": "Node", "multi": true, "name": "node", - "query": "label_values(opensearch_os_cpu_percent{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"}, node)", + "query": "label_values(opensearch_os_cpu_percent{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"}, node)", "refresh": 2, "sort": 1, "type": "query" + }, + { + "hide": 2, + "label": "Loki data source", + "name": "loki_datasource", + "query": "loki", + "regex": "", + "type": "datasource" } ] }, diff --git a/assets/opensearch/dashboards/opensearch-cluster-overview.json b/assets/opensearch/dashboards/opensearch-cluster-overview.json index c694849..27af57f 100644 --- a/assets/opensearch/dashboards/opensearch-cluster-overview.json +++ b/assets/opensearch/dashboards/opensearch-cluster-overview.json @@ -87,9 +87,9 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "max by (job,cluster,node,node,nodeid,role,primary_ip) (last_over_time(opensearch_node_role_bool{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"}[1d]))", + "expr": "max by (job,opensearch_cluster,node,node,nodeid,role,primary_ip) (last_over_time(opensearch_node_role_bool{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"}[1d]))", "instant": true, "legendFormat": "{{node}}" } @@ -115,7 +115,6 @@ }, "indexByName": { "Time": 0, - "cluster": 3, "cluster_manager": 108, "data": 105, "ingest": 106, @@ -123,6 +122,7 @@ "master": 104, "node": 3, "nodeid": 3, + "opensearch_cluster": 3, "remote_cluster_client": 107 }, "renameByName": { @@ -143,7 +143,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The overall health and availability of the OpenSearch cluster.", "fieldConfig": { @@ -219,12 +219,12 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "min by(job,cluster) (opensearch_cluster_status{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"})", + "expr": "min by(job,opensearch_cluster) (opensearch_cluster_status{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{job}}/{{cluster}}" + "legendFormat": "{{job}}/{{opensearch_cluster}}" } ], "title": "Cluster status", @@ -232,7 +232,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The number of running nodes across the OpenSearch cluster.", "fieldConfig": { @@ -286,12 +286,12 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "min by(job,cluster) (opensearch_cluster_nodes_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"})", + "expr": "min by(job,opensearch_cluster) (opensearch_cluster_nodes_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{job}}/{{cluster}}" + "legendFormat": "{{job}}/{{opensearch_cluster}}" } ], "title": "Node count", @@ -299,7 +299,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The number of data nodes in the OpenSearch cluster.", "fieldConfig": { @@ -353,12 +353,12 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "min by(job,cluster) (opensearch_cluster_datanodes_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"})", + "expr": "min by(job,opensearch_cluster) (opensearch_cluster_datanodes_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{job}}/{{cluster}}" + "legendFormat": "{{job}}/{{opensearch_cluster}}" } ], "title": "Data node count", @@ -366,7 +366,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The number of shards in the OpenSearch cluster across all indices.", "fieldConfig": { @@ -420,12 +420,12 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum(max by (type) (opensearch_cluster_shards_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"}))", + "expr": "sum(max by (type) (opensearch_cluster_shards_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"}))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{job}}/{{cluster}}" + "legendFormat": "{{job}}/{{opensearch_cluster}}" } ], "title": "Shard count", @@ -433,7 +433,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Percent of active shards across the OpenSearch cluster.", "fieldConfig": { @@ -492,12 +492,12 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "min by(job,cluster) (opensearch_cluster_shards_active_percent{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"})", + "expr": "min by(job,opensearch_cluster) (opensearch_cluster_shards_active_percent{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{job}}/{{cluster}}" + "legendFormat": "{{job}}/{{opensearch_cluster}}" } ], "title": "Active shards %", @@ -562,41 +562,41 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", role=\"data\"}[1m]) == 1\n) * 2\n", + "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", role=\"data\"}[1m]) == 1\n) * 2\n", "legendFormat": "{{node}}" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", role=\"master\"}[1m]) == 1\n) * 3\n", + "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", role=\"master\"}[1m]) == 1\n) * 3\n", "legendFormat": "{{node}}" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", role=\"ingest\"}[1m]) == 1\n) * 4\n", + "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", role=\"ingest\"}[1m]) == 1\n) * 4\n", "legendFormat": "{{node}}" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", role=\"cluster_manager\"}[1m]) == 1\n) * 5\n", + "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", role=\"cluster_manager\"}[1m]) == 1\n) * 5\n", "legendFormat": "{{node}}" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", role=\"remote_cluster_client\"}[1m]) == 1\n) * 6\n", + "expr": "max by (node,role) (\n max_over_time(opensearch_node_role_bool{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", role=\"remote_cluster_client\"}[1m]) == 1\n) * 6\n", "legendFormat": "{{node}}" } ], @@ -605,7 +605,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Top nodes by OS CPU usage across the OpenSearch cluster.", "fieldConfig": { @@ -658,9 +658,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "topk(10, sort_desc(sum by(node, job,cluster) (opensearch_os_cpu_percent{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"})))", + "expr": "topk(10, sort_desc(sum by(node, job,opensearch_cluster) (opensearch_os_cpu_percent{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"})))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}" @@ -671,7 +671,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The total count of circuit breakers tripped across the OpenSearch cluster.", "fieldConfig": { @@ -722,9 +722,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by(job,cluster, node) (increase(opensearch_circuitbreaker_tripped_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"}[$__interval:]))", + "expr": "sum by(job,opensearch_cluster, node) (increase(opensearch_circuitbreaker_tripped_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"}[$__interval:]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -736,7 +736,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Shard status counts across the Opensearch cluster.", "fieldConfig": { @@ -787,9 +787,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "min by(type, job,cluster) (opensearch_cluster_shards_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"})", + "expr": "min by(type, job,opensearch_cluster) (opensearch_cluster_shards_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}" @@ -800,7 +800,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Top nodes by disk usage across the OpenSearch cluster.", "fieldConfig": { @@ -853,9 +853,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "topk(10, sort_desc((100 * (sum by(node, job,cluster) (opensearch_fs_path_total_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"})- sum by(node, job,cluster) (opensearch_fs_path_free_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"})) / sum by(node, job,cluster) (opensearch_fs_path_total_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"}))))", + "expr": "topk(10, sort_desc((100 * (sum by(node, job,opensearch_cluster) (opensearch_fs_path_total_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"})- sum by(node, job,opensearch_cluster) (opensearch_fs_path_free_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"})) / sum by(node, job,opensearch_cluster) (opensearch_fs_path_total_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"}))))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}" @@ -866,7 +866,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The total count of documents indexed across the OpenSearch cluster.", "fieldConfig": { @@ -940,12 +940,12 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster) (opensearch_indices_indexing_index_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"})", + "expr": "avg by(job,opensearch_cluster) (opensearch_indices_indexing_index_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{job}}/{{cluster}}" + "legendFormat": "{{job}}/{{opensearch_cluster}}" } ], "title": "Total documents", @@ -953,7 +953,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The number of tasks waiting to be executed across the OpenSearch cluster.", "fieldConfig": { @@ -1027,12 +1027,12 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster) (opensearch_cluster_pending_tasks_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"})", + "expr": "avg by(job,opensearch_cluster) (opensearch_cluster_pending_tasks_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{job}}/{{cluster}}" + "legendFormat": "{{job}}/{{opensearch_cluster}}" } ], "title": "Pending tasks", @@ -1040,7 +1040,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The total size of the store across the OpenSearch cluster.", "fieldConfig": { @@ -1114,12 +1114,12 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster) (opensearch_indices_store_size_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"})", + "expr": "avg by(job,opensearch_cluster) (opensearch_indices_store_size_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{job}}/{{cluster}}" + "legendFormat": "{{job}}/{{opensearch_cluster}}" } ], "title": "Store size", @@ -1127,7 +1127,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The max wait time for tasks to be executed across the OpenSearch cluster.", "fieldConfig": { @@ -1201,12 +1201,12 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "max by(job,cluster) (opensearch_cluster_task_max_waiting_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"})", + "expr": "max by(job,opensearch_cluster) (opensearch_cluster_task_max_waiting_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{job}}/{{cluster}}" + "legendFormat": "{{job}}/{{opensearch_cluster}}" } ], "title": "Max task wait time", @@ -1215,7 +1215,7 @@ { "collapsed": false, "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "gridPos": { "h": 1, @@ -1230,7 +1230,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Top indices by combined fetch, query, and scroll request rate across the OpenSearch cluster.", "fieldConfig": { @@ -1308,9 +1308,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "topk(10, sort_desc(avg by(index, job,cluster) (\n opensearch_index_search_fetch_current_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"} + \n opensearch_index_search_query_current_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"} + \n opensearch_index_search_scroll_current_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"}\n)))\n", + "expr": "topk(10, sort_desc(avg by(index, job,opensearch_cluster) (\n opensearch_index_search_fetch_current_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"} + \n opensearch_index_search_query_current_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"} + \n opensearch_index_search_scroll_current_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"}\n)))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -1321,7 +1321,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Top indices by combined fetch, query, and scroll latency across the OpenSearch cluster.", "fieldConfig": { @@ -1395,9 +1395,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "topk(10, sort_desc(sum by(index, job,cluster) ((increase(opensearch_index_search_fetch_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"}[$__interval:])\n+increase(opensearch_index_search_query_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"}[$__interval:])\n+increase(opensearch_index_search_scroll_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"}[$__interval:]))\n/ clamp_min(increase(opensearch_index_search_fetch_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"}[$__interval:])\n+increase(opensearch_index_search_query_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"}[$__interval:])\n+increase(opensearch_index_search_scroll_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"}[$__interval:]), 1))))\n", + "expr": "topk(10, sort_desc(sum by(index, job,opensearch_cluster) ((increase(opensearch_index_search_fetch_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"}[$__interval:])\n+increase(opensearch_index_search_query_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"}[$__interval:])\n+increase(opensearch_index_search_scroll_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"}[$__interval:]))\n/ clamp_min(increase(opensearch_index_search_fetch_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"}[$__interval:])\n+increase(opensearch_index_search_query_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"}[$__interval:])\n+increase(opensearch_index_search_scroll_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"}[$__interval:]), 1))))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1409,7 +1409,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Top indices by cache hit ratio for the combined request and query cache across the OpenSearch cluster.", "fieldConfig": { @@ -1487,9 +1487,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "topk(10, sort_desc(avg by(index, job,cluster) (\n 100 * (opensearch_index_requestcache_hit_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"} + \n opensearch_index_querycache_hit_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"}) / \n clamp_min((opensearch_index_requestcache_hit_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"} + \n opensearch_index_querycache_hit_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"} + \n opensearch_index_requestcache_miss_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"} + \n opensearch_index_querycache_miss_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"}), 1\n ))))\n", + "expr": "topk(10, sort_desc(avg by(index, job,opensearch_cluster) (\n 100 * (opensearch_index_requestcache_hit_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"} + \n opensearch_index_querycache_hit_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"}) / \n clamp_min((opensearch_index_requestcache_hit_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"} + \n opensearch_index_querycache_hit_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"} + \n opensearch_index_requestcache_miss_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"} + \n opensearch_index_querycache_miss_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"}), 1\n ))))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -1500,7 +1500,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Top nodes by rate of ingest across the OpenSearch cluster.", "fieldConfig": { @@ -1578,9 +1578,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "topk(10, sum by(node, job,cluster) (rate(opensearch_ingest_total_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"}[$__rate_interval])))", + "expr": "topk(10, sum by(node, job,opensearch_cluster) (rate(opensearch_ingest_total_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"}[$__rate_interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}" @@ -1591,7 +1591,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Top nodes by ingestion latency across the OpenSearch cluster.", "fieldConfig": { @@ -1669,9 +1669,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "topk(10, sum by(job,cluster, node) (\n increase(opensearch_ingest_total_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"}[$__interval:]) / \n clamp_min(increase(opensearch_ingest_total_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"}[$__interval:]), 1)))\n", + "expr": "topk(10, sum by(job,opensearch_cluster, node) (\n increase(opensearch_ingest_total_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"}[$__interval:]) / \n clamp_min(increase(opensearch_ingest_total_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"}[$__interval:]), 1)))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1683,7 +1683,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Top nodes by ingestion failures across the OpenSearch cluster.", "fieldConfig": { @@ -1757,9 +1757,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "topk(10, sum by(job,cluster, node) (increase(opensearch_ingest_total_failed_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"}[$__interval:])))", + "expr": "topk(10, sum by(job,opensearch_cluster, node) (increase(opensearch_ingest_total_failed_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"}[$__interval:])))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1771,7 +1771,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Top indices by rate of document indexing across the OpenSearch cluster.", "fieldConfig": { @@ -1845,9 +1845,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "topk(10, avg by(job,cluster, index) (opensearch_index_indexing_index_current_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"}))", + "expr": "topk(10, avg by(job,opensearch_cluster, index) (opensearch_index_indexing_index_current_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"}))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -1858,7 +1858,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Top indices by indexing latency across the OpenSearch cluster.", "fieldConfig": { @@ -1932,9 +1932,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "topk(10, avg by(job,cluster, index) \n(increase(opensearch_index_indexing_index_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"}[$__interval:]) / \nclamp_min(increase(opensearch_index_indexing_index_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\", context=\"total\"}[$__interval:]), 1)))\n", + "expr": "topk(10, avg by(job,opensearch_cluster, index) \n(increase(opensearch_index_indexing_index_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"}[$__interval:]) / \nclamp_min(increase(opensearch_index_indexing_index_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\", context=\"total\"}[$__interval:]), 1)))\n", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1946,7 +1946,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Top indices by index document failures across the OpenSearch cluster.", "fieldConfig": { @@ -2020,9 +2020,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "topk(10, avg by(job,cluster, index) (increase(opensearch_index_indexing_index_failed_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"}[$__interval:])))", + "expr": "topk(10, avg by(job,opensearch_cluster, index) (increase(opensearch_index_indexing_index_failed_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"}[$__interval:])))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -2041,8 +2041,8 @@ "templating": { "list": [ { - "label": "Data source", - "name": "datasource", + "label": "Prometheus data source", + "name": "prometheus_datasource", "query": "prometheus", "regex": "", "type": "datasource" @@ -2051,13 +2051,13 @@ "allValue": ".+", "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "includeAll": true, "label": "Job", "multi": true, "name": "job", - "query": "label_values(opensearch_cluster_status{cluster!=\"\"}, job)", + "query": "label_values(opensearch_cluster_status{opensearch_cluster!=\"\"}, job)", "refresh": 2, "sort": 1, "type": "query" @@ -2066,13 +2066,13 @@ "allValue": ".+", "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "includeAll": true, - "label": "Cluster", + "label": "Opensearch_cluster", "multi": true, - "name": "cluster", - "query": "label_values(opensearch_cluster_status{cluster!=\"\",job=~\"$job\"}, cluster)", + "name": "opensearch_cluster", + "query": "label_values(opensearch_cluster_status{opensearch_cluster!=\"\",job=~\"$job\"}, opensearch_cluster)", "refresh": 2, "sort": 1, "type": "query" diff --git a/assets/opensearch/dashboards/search-and-index-overview.json b/assets/opensearch/dashboards/search-and-index-overview.json index b711465..6e5ce4a 100644 --- a/assets/opensearch/dashboards/search-and-index-overview.json +++ b/assets/opensearch/dashboards/search-and-index-overview.json @@ -15,7 +15,7 @@ { "collapsed": false, "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "gridPos": { "h": 1, @@ -30,7 +30,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Rate of fetch, scroll, and query requests by selected index.", "fieldConfig": { @@ -129,27 +129,27 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by (job,cluster,index) (opensearch_index_search_query_current_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=~\"total\"})", + "expr": "avg by (job,opensearch_cluster,index) (opensearch_index_search_query_current_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=~\"total\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}} - query" }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by (job,cluster,index) (opensearch_index_search_fetch_current_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=~\"total\"})", + "expr": "avg by (job,opensearch_cluster,index) (opensearch_index_search_fetch_current_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=~\"total\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}} - fetch" }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by (job,cluster,index) (opensearch_index_search_scroll_current_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=~\"total\"})", + "expr": "avg by (job,opensearch_cluster,index) (opensearch_index_search_scroll_current_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=~\"total\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}} - scroll" @@ -160,7 +160,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Latency of fetch, scroll, and query requests by selected index.", "fieldConfig": { @@ -259,9 +259,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by (job,cluster,index) (increase(opensearch_index_search_query_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\"}[$__interval:]) / clamp_min(increase(opensearch_index_search_query_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:]), 1))", + "expr": "avg by (job,opensearch_cluster,index) (increase(opensearch_index_search_query_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\"}[$__interval:]) / clamp_min(increase(opensearch_index_search_query_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:]), 1))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -269,9 +269,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by (job,cluster,index) (increase(opensearch_index_search_fetch_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:]) / clamp_min(increase(opensearch_index_search_fetch_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:]), 1))", + "expr": "avg by (job,opensearch_cluster,index) (increase(opensearch_index_search_fetch_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:]) / clamp_min(increase(opensearch_index_search_fetch_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:]), 1))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -279,9 +279,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by (job,cluster,index) (increase(opensearch_index_search_scroll_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:]) / clamp_min(increase(opensearch_index_search_scroll_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:]), 1))", + "expr": "avg by (job,opensearch_cluster,index) (increase(opensearch_index_search_scroll_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:]) / clamp_min(increase(opensearch_index_search_scroll_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:]), 1))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -293,7 +293,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Ratio of query cache and request cache hits and misses.", "fieldConfig": { @@ -392,18 +392,18 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (100 * (opensearch_index_requestcache_hit_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}) / clamp_min(opensearch_index_requestcache_hit_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"} + opensearch_index_requestcache_miss_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}, 1))", + "expr": "avg by(job,opensearch_cluster,index) (100 * (opensearch_index_requestcache_hit_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}) / clamp_min(opensearch_index_requestcache_hit_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"} + opensearch_index_requestcache_miss_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}, 1))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}} - request" }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (100 * (opensearch_index_querycache_hit_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}) / clamp_min(opensearch_index_querycache_hit_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"} + opensearch_index_querycache_miss_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}, 1))", + "expr": "avg by(job,opensearch_cluster,index) (100 * (opensearch_index_querycache_hit_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}) / clamp_min(opensearch_index_querycache_hit_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"} + opensearch_index_querycache_miss_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}, 1))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}} - query" @@ -414,7 +414,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Total evictions count by cache type for the selected index.", "fieldConfig": { @@ -513,9 +513,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (increase(opensearch_index_querycache_evictions_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:]))", + "expr": "avg by(job,opensearch_cluster,index) (increase(opensearch_index_querycache_evictions_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -523,9 +523,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (increase(opensearch_index_requestcache_evictions_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:]))", + "expr": "avg by(job,opensearch_cluster,index) (increase(opensearch_index_requestcache_evictions_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -533,9 +533,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (increase(opensearch_index_fielddata_evictions_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:]))", + "expr": "avg by(job,opensearch_cluster,index) (increase(opensearch_index_fielddata_evictions_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -548,7 +548,7 @@ { "collapsed": false, "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "gridPos": { "h": 1, @@ -563,7 +563,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Rate of indexed documents for the selected index.", "fieldConfig": { @@ -662,9 +662,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (opensearch_index_indexing_index_current_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"})", + "expr": "avg by(job,opensearch_cluster,index) (opensearch_index_indexing_index_current_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -675,7 +675,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Document indexing latency for the selected index.", "fieldConfig": { @@ -774,9 +774,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (increase(opensearch_index_indexing_index_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=~\"total\"}[$__interval:]) / clamp_min(increase(opensearch_index_indexing_index_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=~\"total\"}[$__interval:]),1))", + "expr": "avg by(job,opensearch_cluster,index) (increase(opensearch_index_indexing_index_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=~\"total\"}[$__interval:]) / clamp_min(increase(opensearch_index_indexing_index_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=~\"total\"}[$__interval:]),1))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -788,7 +788,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Number of indexing failures for the selected index.", "fieldConfig": { @@ -887,9 +887,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (increase(opensearch_index_indexing_index_failed_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:]))", + "expr": "avg by(job,opensearch_cluster,index) (increase(opensearch_index_indexing_index_failed_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:]))", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -901,7 +901,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Index flush latency for the selected index.", "fieldConfig": { @@ -1000,9 +1000,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (increase(opensearch_index_flush_total_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:]) / clamp_min(increase(opensearch_index_flush_total_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:]),1))", + "expr": "avg by(job,opensearch_cluster,index) (increase(opensearch_index_flush_total_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:]) / clamp_min(increase(opensearch_index_flush_total_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:]),1))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -1013,7 +1013,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Index merge time for the selected index.", "fieldConfig": { @@ -1112,27 +1112,27 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (increase(opensearch_index_merges_total_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:])) > 0", + "expr": "avg by(job,opensearch_cluster,index) (increase(opensearch_index_merges_total_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}} - total" }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (increase(opensearch_index_merges_total_stopped_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:])) > 0", + "expr": "avg by(job,opensearch_cluster,index) (increase(opensearch_index_merges_total_stopped_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}} - stopped" }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (increase(opensearch_index_merges_total_throttled_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:])) > 0", + "expr": "avg by(job,opensearch_cluster,index) (increase(opensearch_index_merges_total_throttled_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}} - throttled" @@ -1143,7 +1143,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Index refresh latency for the selected index.", "fieldConfig": { @@ -1242,9 +1242,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (increase(opensearch_index_refresh_total_time_seconds{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:]) / clamp_min(increase(opensearch_index_refresh_total_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:]),1))", + "expr": "avg by(job,opensearch_cluster,index) (increase(opensearch_index_refresh_total_time_seconds{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:]) / clamp_min(increase(opensearch_index_refresh_total_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:]),1))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -1255,7 +1255,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Current number of translog operations for the selected index.", "fieldConfig": { @@ -1354,9 +1354,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (opensearch_index_translog_operations_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"})", + "expr": "avg by(job,opensearch_cluster,index) (opensearch_index_translog_operations_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -1367,7 +1367,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Rate of documents deleted for the selected index.", "fieldConfig": { @@ -1466,9 +1466,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by (job,cluster,index) (opensearch_index_indexing_delete_current_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"})", + "expr": "avg by (job,opensearch_cluster,index) (opensearch_index_indexing_delete_current_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -1480,7 +1480,7 @@ { "collapsed": false, "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "gridPos": { "h": 1, @@ -1495,7 +1495,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Number of indexed documents for the selected index.", "fieldConfig": { @@ -1594,9 +1594,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by (job,cluster,index) (opensearch_index_indexing_index_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"})", + "expr": "avg by (job,opensearch_cluster,index) (opensearch_index_indexing_index_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -1607,7 +1607,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Current number of segments for the selected index.", "fieldConfig": { @@ -1706,9 +1706,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (opensearch_index_segments_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"})", + "expr": "avg by(job,opensearch_cluster,index) (opensearch_index_segments_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -1719,7 +1719,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Number of merge operations for the selected index.", "fieldConfig": { @@ -1818,9 +1818,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (increase(opensearch_index_merges_total_docs_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}[$__interval:])) > 0", + "expr": "avg by(job,opensearch_cluster,index) (increase(opensearch_index_merges_total_docs_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}[$__interval:])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -1831,7 +1831,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Size of query cache and request cache.", "fieldConfig": { @@ -1930,18 +1930,18 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (opensearch_index_querycache_memory_size_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"})", + "expr": "avg by(job,opensearch_cluster,index) (opensearch_index_querycache_memory_size_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}} - query" }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (opensearch_index_requestcache_memory_size_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"})", + "expr": "avg by(job,opensearch_cluster,index) (opensearch_index_requestcache_memory_size_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}} - request" @@ -1952,7 +1952,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Size of the store in bytes for the selected index.", "fieldConfig": { @@ -2051,9 +2051,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (opensearch_index_store_size_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"})", + "expr": "avg by(job,opensearch_cluster,index) (opensearch_index_store_size_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -2064,7 +2064,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Memory used by segments for the selected index.", "fieldConfig": { @@ -2163,9 +2163,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (opensearch_index_segments_memory_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"})", + "expr": "avg by(job,opensearch_cluster,index) (opensearch_index_segments_memory_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -2176,7 +2176,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "Size of merge operations in bytes for the selected index.", "fieldConfig": { @@ -2275,9 +2275,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "avg by(job,cluster,index) (opensearch_index_merges_current_size_bytes{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", context=\"total\"}) > 0", + "expr": "avg by(job,opensearch_cluster,index) (opensearch_index_merges_current_size_bytes{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", context=\"total\"}) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -2288,7 +2288,7 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "description": "The number of index shards for the selected index.", "fieldConfig": { @@ -2387,9 +2387,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, - "expr": "sum by (index) (avg by(job,cluster,index) (opensearch_index_shards_number{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\",index=~\"$index\", type=~\"active|active_primary\"}))", + "expr": "sum by (index) (avg by(job,opensearch_cluster,index) (opensearch_index_shards_number{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\",index=~\"$index\", type=~\"active|active_primary\"}))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{index}}" @@ -2407,8 +2407,8 @@ "templating": { "list": [ { - "label": "Data source", - "name": "datasource", + "label": "Prometheus data source", + "name": "prometheus_datasource", "query": "prometheus", "regex": "", "type": "datasource" @@ -2417,13 +2417,13 @@ "allValue": ".+", "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "includeAll": true, "label": "Job", "multi": true, "name": "job", - "query": "label_values(opensearch_index_search_fetch_count{cluster!=\"\"}, job)", + "query": "label_values(opensearch_index_search_fetch_count{opensearch_cluster!=\"\"}, job)", "refresh": 2, "sort": 1, "type": "query" @@ -2432,13 +2432,13 @@ "allValue": ".+", "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "includeAll": true, - "label": "Cluster", + "label": "Opensearch_cluster", "multi": true, - "name": "cluster", - "query": "label_values(opensearch_index_search_fetch_count{cluster!=\"\",job=~\"$job\"}, cluster)", + "name": "opensearch_cluster", + "query": "label_values(opensearch_index_search_fetch_count{opensearch_cluster!=\"\",job=~\"$job\"}, opensearch_cluster)", "refresh": 2, "sort": 1, "type": "query" @@ -2447,13 +2447,13 @@ "allValue": ".+", "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${prometheus_datasource}" }, "includeAll": true, "label": "Index", "multi": true, "name": "index", - "query": "label_values(opensearch_index_search_fetch_count{cluster!=\"\",job=~\"$job\",cluster=~\"$cluster\"}, index)", + "query": "label_values(opensearch_index_search_fetch_count{opensearch_cluster!=\"\",job=~\"$job\",opensearch_cluster=~\"$opensearch_cluster\"}, index)", "refresh": 2, "sort": 1, "type": "query" diff --git a/assets/openstack/alerts.yaml b/assets/openstack/alerts.yaml index 7a31ac0..83d61af 100644 --- a/assets/openstack/alerts.yaml +++ b/assets/openstack/alerts.yaml @@ -1,57 +1,218 @@ groups: - name: openstack-alerts-openstack rules: - - alert: OpenStackPlacementHighMemoryUsageWarning + - alert: OpenStackGlanceIsDown annotations: - description: The cloud on instance {{$labels.instance}} is using {{ printf "%.0f" - $value }} percent of its allocated memory, which is above the threshold of - 80 percent. - summary: The cloud is using a significant percentage of its allocated memory. + description: OpenStack Glance service is down on cluster {{ $labels.instance + }}. + summary: OpenStack Glance is down. expr: | - 100 * openstack_placement_resource_usage{job=~"integrations/openstack", resourcetype="MEMORY_MB"} / clamp_min(openstack_placement_resource_total{job=~"integrations/openstack", resourcetype="MEMORY_MB"}, 1) > 80 + openstack_glance_up{job=~"integrations/openstack"} == 0 for: 5m labels: - severity: warning - - alert: OpenStackPlacementHighMemoryUsageCritical + severity: critical + - alert: OpenStackHeatIsDown annotations: - description: The cloud on instance {{$labels.instance}} is using {{ printf "%.0f" - $value }} percent of its allocated memory, which is above the threshold of - 90 percent. - summary: The cloud is using a large percentage of its allocated memory, consider + description: OpenStack Heat service is down on cluster {{ $labels.instance }}. + summary: OpenStack Heat is down. + expr: | + openstack_heat_up{job=~"integrations/openstack"} == 0 + for: 5m + labels: + severity: critical + - alert: OpenStackIdentityIsDown + annotations: + description: OpenStack Identity service is down on cluster {{ $labels.instance + }}. + summary: OpenStack Identity is down. + expr: | + openstack_identity_up{job=~"integrations/openstack"} == 0 + for: 5m + labels: + severity: critical + - alert: OpenStackPlacementIsDown + annotations: + description: OpenStack Placement service is down on cluster {{ $labels.instance + }}. + summary: OpenStack Placement is down. + expr: | + openstack_placement_up{job=~"integrations/openstack"} == 0 + for: 5m + labels: + severity: critical + - alert: OpenStackPlacementHighMemoryUsageWarning + annotations: + description: | + OpenStack {{$labels.instance}} is using {{ printf "%.0f" $value }} percent of its allocated memory, + which is above the threshold of 80 percent. + summary: OpenStack is using a significant percentage of its allocated memory. + expr: | + 100 * sum by (job,instance) (openstack_placement_resource_usage{job=~"integrations/openstack", resourcetype="MEMORY_MB"}) + / + (sum by (job,instance) (openstack_placement_resource_total{job=~"integrations/openstack", resourcetype="MEMORY_MB"}) > 0) + > 80 + for: 5m + keep_firing_for: 5m + labels: + severity: warning + - alert: OpenStackNovaAgentDown + annotations: + description: | + OpenStack {{$labels.instance}} is using {{ printf "%.0f" $value }} percent of its allocated memory, + which is above the threshold of 90 percent. + summary: OpenStack is using a large percentage of its allocated memory, consider allocating more resources. - expr: "100 * openstack_placement_resource_usage{job=~\"integrations/openstack\", - resourcetype=\"MEMORY_MB\"} / clamp_min(openstack_placement_resource_total{job=~\"integrations/openstack\", - resourcetype=\"MEMORY_MB\"}, 1) > 90 \n" + expr: | + 100 * sum by (job,instance) (openstack_placement_resource_usage{job=~"integrations/openstack", resourcetype="MEMORY_MB"}) + / + (sum by (job,instance) (openstack_placement_resource_total{job=~"integrations/openstack", resourcetype="MEMORY_MB"}) > 0) + > 90 + for: 5m + keep_firing_for: 5m + labels: + severity: critical + - alert: OpenStackPlacementHighVCPUUsageWarning + annotations: + description: | + OpenStack {{$labels.instance}} is using {{ printf "%.0f" $value }} percent of its allocated vCPU, + which is above the threshold of 80 percent. + summary: OpenStack is using a significant percentage of its allocated vCPU. + expr: | + 100 * sum by (job,instance) (openstack_placement_resource_usage{job=~"integrations/openstack", resourcetype="VCPU"}) + / + (sum by (job,instance) (openstack_placement_resource_total{job=~"integrations/openstack", resourcetype="VCPU"}) > 0) + > 80 + for: 5m + keep_firing_for: 5m + labels: + severity: warning + - alert: OpenStackPlacementHighVCPUUsageCritical + annotations: + description: | + OpenStack {{$labels.instance}} is using {{ printf "%.0f" $value }} percent of its allocated vCPU, + which is above the threshold of 90 percent. + summary: OpenStack is using a large percentage of its allocated vCPU, consider + allocating more resources. + expr: | + 100 * sum by (job,instance) (openstack_placement_resource_usage{job=~"integrations/openstack", resourcetype="VCPU"}) + / + (sum by (job,instance) (openstack_placement_resource_total{job=~"integrations/openstack", resourcetype="VCPU"}) > 0) + > 90 + for: 5m + keep_firing_for: 5m + labels: + severity: critical + - alert: OpenStackNeutronHighIPsUsageWarning + annotations: + description: | + Network {{$labels.network_name}} is running out of free IP addresses on OpenStack {{$labels.instance}}, + {{ printf "%.0f" $value }} percent of the pool used, + {{ with printf `sum(openstack_neutron_network_ip_availabilities_total{job=~"integrations/openstack", instance=~"%s", network_name=~"%s"}) - (sum(openstack_neutron_network_ip_availabilities_used{job=~"integrations/openstack", instance=~"%s", network_name=~"%s"}))` .Labels.instance .Labels.network_name .Labels.instance .Labels.network_name | query -}}{{ . | first | value | humanize }}{{ end }} IP addresses available. + summary: Free IP addresses are running out. + expr: "100 * \nsum by (job,instance, network_name) (openstack_neutron_network_ip_availabilities_used{job=~\"integrations/openstack\", + network_name=~\".+\"}) \n/\n(sum by (job,instance, network_name) (openstack_neutron_network_ip_availabilities_total{job=~\"integrations/openstack\", + network_name=~\".+\"})\n> 0)\n> 80\n" + for: 5m + keep_firing_for: 5m + labels: + severity: warning + - alert: OpenStackNeutronHighIPsUsageCritical + annotations: + description: | + Network {{$labels.network_name}} is running out of free IP addresses on OpenStack {{$labels.instance}}, + {{ printf "%.0f" $value }} percent of the pool used, + {{ with printf `sum(openstack_neutron_network_ip_availabilities_total{job=~"integrations/openstack", instance=~"%s", network_name=~"%s"}) - (sum(openstack_neutron_network_ip_availabilities_used{job=~"integrations/openstack", instance=~"%s", network_name=~"%s"}))` .Labels.instance .Labels.network_name .Labels.instance .Labels.network_name | query -}}{{ . | first | value | humanize }}{{ end }} IP addresses available. + summary: There are practically no free IP addresses left. + expr: "100 * \nsum by (job,instance, network_name) (openstack_neutron_network_ip_availabilities_used{job=~\"integrations/openstack\", + network_name=~\".+\"}) \n/\n(sum by (job,instance, network_name) (openstack_neutron_network_ip_availabilities_total{job=~\"integrations/openstack\", + network_name=~\".+\"})\n> 0)\n> 90\n" + for: 5m + keep_firing_for: 5m + labels: + severity: critical +- name: openstack-nova-alertsopenstack + rules: + - alert: OpenStackNovaIsDown + annotations: + description: OpenStack Nova is down on {{ $labels.instance }}. + summary: OpenStack Nova service is down. + expr: | + openstack_nova_up{job=~"integrations/openstack"} == 0 + for: 5m + labels: + severity: critical + - alert: OpenStackNovaAgentIsDown + annotations: + description: An OpenStack Nova agent is down on hostname {{ $labels.hostname + }} on OpenStack cluster {{ $labels.instance }}. + summary: OpenStack Nova agent is down on the specific node. + expr: | + openstack_nova_agent_state{job=~"integrations/openstack",adminState="enabled"} != 1 for: 5m labels: severity: critical - alert: OpenStackNovaHighVMMemoryUsage annotations: - description: Virtual machines on the cloud on {{$labels.instance}} are using - {{ printf "%.0f" $value }} percent of their allocated memory, which is above - the threshold of 80 percent. + description: | + Virtual machines on OpenStack {{ $labels.instance }} are using {{ printf "%.0f" $value }} percent of their allocated memory, + which is above the threshold of 80 percent. summary: VMs are using a high percentage of their allocated memory. expr: | - 100 * openstack_nova_limits_memory_used{job=~"integrations/openstack"} / clamp_min(openstack_nova_limits_memory_max{job=~"integrations/openstack"}, 1) > 80 + 100 * openstack_nova_limits_memory_used{job=~"integrations/openstack"} / (openstack_nova_limits_memory_max{job=~"integrations/openstack"} > 0) > 80 for: 5m labels: severity: warning - alert: OpenStackNovaHighVMVCPUUsage annotations: - description: Virtual machines on the cloud on {{$labels.instance}} are using - {{ printf "%.0f" $value }} percent of their allocated virtual CPUs, which - is above the threshold of 80 percent. + description: | + Virtual machines on OpenStack {{$labels.instance}} are using {{ printf "%.0f" $value }} percent of their allocated virtual CPUs, + which is above the threshold of 80 percent. summary: VMs are using a high percentage of their allocated virtual CPUs. expr: | - 100 * openstack_nova_limits_vcpus_used{job=~"integrations/openstack"} / clamp_min(openstack_nova_limits_vcpus_max{job=~"integrations/openstack"}, 1) > 80 + 100 * openstack_nova_limits_vcpus_used{job=~"integrations/openstack"} / (openstack_nova_limits_vcpus_max{job=~"integrations/openstack"} > 0) > 80 for: 5m labels: severity: warning +- name: openstack-neutron-alertsopenstack + rules: + - alert: OpenStackNeutronIsDown + annotations: + description: OpenStack Neutron service is down on cluster {{ $labels.instance + }}. + summary: OpenStack Neutron is down. + expr: | + openstack_neutron_up{job=~"integrations/openstack"} == 0 + for: 5m + labels: + severity: critical + - alert: OpenStackNeutronAgentIsDown + annotations: + description: | + OpenStack Neutron agent`s service {{ $labels.service }} is down on hostname {{ $labels.hostname }} on OpenStack cluster {{ $labels.instance }}. + If {{ $labels.service }} is no longer required on this host, disable it administratively by running: + OpenStack network agent set {{ $labels.id }} --disable + runbook_url: https://docs.openstack.org/neutron/zed/admin/config-services-agent.html#agent-s-admin-state-specific-config-options + summary: OpenStack Neutron agent is down on the specific node. + expr: | + openstack_neutron_agent_state{job=~"integrations/openstack",adminState="up"} != 1 + for: 5m + labels: + severity: critical + - alert: OpenStackNeutronL3AgentIsDown + annotations: + description: OpenStack Neutron L3 agent is down on hostname {{ $labels.agent_host + }} on OpenStack cluster {{ $labels.instance }}. + summary: OpenStack Neutron L3 agent is down on the specific node. + expr: | + openstack_neutron_l3_agent_of_router{job=~"integrations/openstack",agent_admin_up="true"} != 1 + for: 5m + labels: + severity: critical - alert: OpenStackNeutronHighDisconnectedPortRate annotations: - description: '{{ printf "%.0f" $value }} percent of ports managed by the Neutron - service on instance {{$labels.instance}} have no IP addresses assigned to - them, which is above the threshold of 25' + description: | + {{ printf "%.0f" $value }} percent of ports managed by the Neutron service on OpenStack cluster {{$labels.instance}} have no IP addresses assigned to them, + which is above the threshold of 25. summary: A high rate of ports have no IP addresses assigned to them. expr: | 100 * openstack_neutron_ports_no_ips{job=~"integrations/openstack"} / clamp_min(openstack_neutron_ports{job=~"integrations/openstack"}, 1) > 25 @@ -60,45 +221,67 @@ groups: severity: critical - alert: OpenStackNeutronHighInactiveRouterRate annotations: - description: '{{ printf "%.0f" $value }} percent of routers managed by the Neutron - service on instance {{$labels.instance}} are currently inactive, which is - above the threshold of 15' + description: | + {{ printf "%.0f" $value }} percent of routers managed by the Neutron service on cluster {{$labels.instance}} are currently inactive, + which is above the threshold of 15. summary: A high rate of routers are currently inactive. expr: | 100 * openstack_neutron_routers_not_active{job=~"integrations/openstack"} / clamp_min(openstack_neutron_routers{job=~"integrations/openstack"}, 1) > 15 for: 5m labels: severity: critical - - alert: OpenStackCinderHighBackupMemoryUsage +- name: openstack-cinder-alertsopenstack + rules: + - alert: OpenStackCinderIsDown annotations: - description: Backups managed by the Cinder service on instance {{$labels.instance}} - are using {{ printf "%.0f" $value }} percent of their allocated memory, which - is above the threshold of 80 percent. - summary: Cinder backups are using a large amount of their maximum memory. + description: OpenStack Cinder service is down on cluster {{ $labels.instance + }}. + summary: OpenStack Cinder is down. expr: | - 100 * openstack_cinder_limits_backup_used_gb{job=~"integrations/openstack"} / clamp_min(openstack_cinder_limits_backup_max_gb{job=~"integrations/openstack"}, 1) > 80 + openstack_cinder_up{job=~"integrations/openstack"} == 0 for: 5m labels: - severity: warning - - alert: OpenStackCinderHighVolumeMemoryUsage + severity: critical + - alert: OpenStackCinderAgentIsDown annotations: - description: Volumes managed by the Cinder service on instance {{$labels.instance}} - are using {{ printf "%.0f" $value }} percent of their allocated memory, which - is above the threshold of 80 percent. - summary: Cinder volumes are using a large amount of their maximum memory. + description: OpenStack Cinder agent is down on hostname {{ $labels.hostname + }} on OpenStack cluster {{ $labels.instance }}. + summary: OpenStack Cinder agent is down on the specific node. expr: | - 100 * openstack_cinder_limits_volume_used_gb{job=~"integrations/openstack"} / clamp_min(openstack_cinder_limits_volume_max_gb{job=~"integrations/openstack"}, 1) > 80 + openstack_cinder_agent_state{job=~"integrations/openstack",adminState="enabled"} != 1 for: 5m labels: - severity: warning + severity: critical - alert: OpenStackCinderHighPoolCapacityUsage annotations: - description: Pools managed by the Cinder service on instance {{$labels.instance}} - are using {{ printf "%.0f" $value }} percent of their allocated capacity, + description: | + Pools managed by the Cinder service on cluster {{$labels.instance}} are using {{ printf "%.0f" $value }} percent of their allocated capacity, which is above the threshold of 80 percent. summary: Cinder pools are using a large amount of their maximum capacity. expr: | 100 * (openstack_cinder_pool_capacity_total_gb{job=~"integrations/openstack"} - openstack_cinder_pool_capacity_free_gb{job=~"integrations/openstack"}) / clamp_min(openstack_cinder_pool_capacity_total_gb{job=~"integrations/openstack"}, 1) > 80 + for: 10m + labels: + severity: warning + - alert: OpenStackCinderHighVolumeMemoryUsage + annotations: + description: | + Volumes managed by the Cinder service on cluster {{$labels.instance}} are using {{ printf "%.0f" $value }} percent of their allocated memory, + which is above the threshold of 80 percent. + summary: Cinder volumes are using a large amount of their maximum memory. + expr: | + 100 * openstack_cinder_limits_volume_used_gb{job=~"integrations/openstack"} / (openstack_cinder_limits_volume_max_gb{job=~"integrations/openstack"} > 0) > 80 + for: 5m + labels: + severity: warning + - alert: OpenStackCinderHighBackupMemoryUsage + annotations: + description: | + Backups managed by the Cinder service on cluster {{$labels.instance}} are using {{ printf "%.0f" $value }} percent of their allocated memory, + which is above the threshold of 80 percent. + summary: Cinder backups are using a large amount of their maximum memory. + expr: | + 100 * openstack_cinder_limits_backup_used_gb{job=~"integrations/openstack"} / (openstack_cinder_limits_backup_max_gb{job=~"integrations/openstack"} > 0) > 80 for: 5m labels: severity: warning diff --git a/assets/openstack/dashboards/cinder b/assets/openstack/dashboards/cinder index 58cdda0..604951f 100644 --- a/assets/openstack/dashboards/cinder +++ b/assets/openstack/dashboards/cinder @@ -68,7 +68,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -116,7 +116,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -171,7 +171,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -237,7 +237,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -294,7 +294,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -350,7 +350,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -400,7 +400,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -578,7 +578,7 @@ "label": "Loki data source", "name": "loki_datasource", "query": "loki", - "regex": "(?!grafanacloud.+usage-insights|grafanacloud.+alert-state-history).+", + "regex": "", "type": "datasource" } ] diff --git a/assets/openstack/dashboards/logs b/assets/openstack/dashboards/logs index 54b789f..d8778b6 100644 --- a/assets/openstack/dashboards/logs +++ b/assets/openstack/dashboards/logs @@ -153,17 +153,19 @@ }, "gridPos": { "h": 6, - "w": 24 + "w": 24, + "x": 0, + "y": 0 }, "id": 1, - "interval": "30s", + "maxDataPoints": 100, "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -193,7 +195,9 @@ }, "gridPos": { "h": 18, - "w": 24 + "w": 24, + "x": 0, + "y": 18 }, "id": 2, "options": { @@ -203,7 +207,7 @@ "showTime": false, "wrapLogMessage": false }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -218,7 +222,7 @@ } ], "refresh": "1m", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "openstack" ], @@ -228,7 +232,7 @@ "label": "Loki data source", "name": "loki_datasource", "query": "loki", - "regex": "(?!grafanacloud.+usage-insights|grafanacloud.+alert-state-history).+", + "regex": "", "type": "datasource" }, { diff --git a/assets/openstack/dashboards/neutron b/assets/openstack/dashboards/neutron index d2aa9de..eac8064 100644 --- a/assets/openstack/dashboards/neutron +++ b/assets/openstack/dashboards/neutron @@ -68,7 +68,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -116,7 +116,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -166,7 +166,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -216,7 +216,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -403,7 +403,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -603,7 +603,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -669,7 +669,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -719,7 +719,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -895,7 +895,7 @@ "label": "Loki data source", "name": "loki_datasource", "query": "loki", - "regex": "(?!grafanacloud.+usage-insights|grafanacloud.+alert-state-history).+", + "regex": "", "type": "datasource" } ] diff --git a/assets/openstack/dashboards/nova b/assets/openstack/dashboards/nova index 4208d29..3923879 100644 --- a/assets/openstack/dashboards/nova +++ b/assets/openstack/dashboards/nova @@ -68,7 +68,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -116,7 +116,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -173,7 +173,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -230,7 +230,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -287,7 +287,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -463,7 +463,7 @@ "label": "Loki data source", "name": "loki_datasource", "query": "loki", - "regex": "(?!grafanacloud.+usage-insights|grafanacloud.+alert-state-history).+", + "regex": "", "type": "datasource" } ] diff --git a/assets/openstack/dashboards/overview b/assets/openstack/dashboards/overview index 75ccffc..48bb104 100644 --- a/assets/openstack/dashboards/overview +++ b/assets/openstack/dashboards/overview @@ -68,7 +68,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -123,7 +123,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -178,7 +178,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -233,7 +233,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -288,7 +288,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -343,7 +343,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -370,7 +370,7 @@ }, "id": 7, "options": { - "alertInstanceLabelFilter": "job=~\"${job:regex}\"" + "alertInstanceLabelFilter": "job=~\"integrations/openstack\"=~\"${job=~\"integrations/openstack\":regex}\",job=~\"${job:regex}\"" }, "pluginVersion": "v10.0.0", "title": "Alerts", @@ -484,7 +484,7 @@ { "id": "joinByField", "options": { - "byField": "instance", + "byField": "Time", "mode": "outer" } }, @@ -504,6 +504,12 @@ "__name__ 4": true, "__name__ 5": true, "__name__ 6": true, + "cluster 1": false, + "cluster 2": true, + "cluster 3": true, + "cluster 4": true, + "cluster 5": true, + "cluster 6": true, "hostname 1": true, "hostname 2": true, "hostname 3": true, @@ -526,43 +532,19 @@ }, "includeByName": { }, "indexByName": { - "Time 1": 34, - "Time 2": 9, - "Time 3": 14, - "Time 4": 19, - "Time 5": 24, - "Time 6": 29, - "Value #A": 3, - "Value #B": 4, - "Value #C": 5, - "Value #D": 6, - "Value #E": 7, - "Value #F": 8, - "__name__ 1": 1, - "__name__ 2": 10, - "__name__ 3": 15, - "__name__ 4": 20, - "__name__ 5": 25, - "__name__ 6": 30, - "hostname 1": 35, - "hostname 2": 11, - "hostname 3": 16, - "hostname 4": 21, - "hostname 5": 26, - "hostname 6": 31, - "instance": 0, - "job 1": 36, - "job 2": 12, - "job 3": 17, - "job 4": 22, - "job 5": 27, - "job 6": 32, - "resourcetype 1": 2, - "resourcetype 2": 13, - "resourcetype 3": 18, - "resourcetype 4": 23, - "resourcetype 5": 28, - "resourcetype 6": 33 + "Time": 7, + "Value #A": 1, + "Value #B": 2, + "Value #C": 3, + "Value #D": 4, + "Value #E": 5, + "Value #F": 6, + "cluster 1": 0, + "cluster 2": 8, + "cluster 3": 9, + "cluster 4": 10, + "cluster 5": 11, + "cluster 6": 12 }, "renameByName": { "Time": "", @@ -572,6 +554,7 @@ "Value #D": "Memory used", "Value #E": "VCPUs available", "Value #F": "VCPUs used", + "cluster 1": "cluster", "hostname": "Hostname", "instance": "Instance", "openstack_placement_resource_total": "Total", @@ -583,15 +566,145 @@ ], "type": "table" }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "max": "150", + "min": "0", + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 99 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 13 + }, + "id": 9, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus_datasource}" + }, + "expr": "(100*sum(openstack_placement_resource_usage{job=~\"$job\",instance=~\"$instance\",resourcetype=\"VCPU\"}))/sum(openstack_placement_resource_total{job=~\"$job\",instance=~\"$instance\",resourcetype=\"VCPU\"})" + } + ], + "title": "vCPU used", + "type": "gauge" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "max": "150", + "min": "0", + "thresholds": { + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 99 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 13 + }, + "id": 10, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus_datasource}" + }, + "expr": "(100*sum(openstack_placement_resource_usage{job=~\"$job\",instance=~\"$instance\",resourcetype=\"MEMORY_MB\"}))/sum(openstack_placement_resource_total{job=~\"$job\",instance=~\"$instance\",resourcetype=\"MEMORY_MB\"})" + } + ], + "title": "Memory used", + "type": "gauge" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 20 + } + ] + } + } + }, + "gridPos": { + "h": 5, + "w": 16, + "x": 8, + "y": 13 + }, + "id": 11, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus_datasource}" + }, + "expr": "openstack_neutron_network_ip_availabilities_total{job=~\"$job\",instance=~\"$instance\",network_name=~\".+\"}-openstack_neutron_network_ip_availabilities_used{job=~\"$job\",instance=~\"$instance\",network_name=~\".+\"}", + "legendFormat": "{{network_name}}" + } + ], + "title": "Free IPs", + "type": "stat" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 0, "x": 24, - "y": 16 + "y": 21 }, - "id": 9, + "id": 12, "title": "Keystone service", "type": "row" }, @@ -613,9 +726,9 @@ "h": 8, "w": 4, "x": 0, - "y": 17 + "y": 22 }, - "id": 10, + "id": 13, "options": { "colorMode": "fixed", "graphMode": "none", @@ -625,7 +738,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -656,9 +769,9 @@ "h": 8, "w": 4, "x": 4, - "y": 17 + "y": 22 }, - "id": 11, + "id": 14, "options": { "colorMode": "fixed", "graphMode": "none", @@ -668,7 +781,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -699,9 +812,9 @@ "h": 8, "w": 4, "x": 8, - "y": 17 + "y": 22 }, - "id": 12, + "id": 15, "options": { "colorMode": "fixed", "graphMode": "none", @@ -711,7 +824,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -746,9 +859,9 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 22 }, - "id": 13, + "id": 16, "options": { "legend": { "calcs": [ ], @@ -759,7 +872,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -825,9 +938,9 @@ "h": 8, "w": 24, "x": 0, - "y": 25 + "y": 30 }, - "id": 14, + "id": 17, "pluginVersion": "v10.0.0", "targets": [ { @@ -894,9 +1007,9 @@ "h": 1, "w": 0, "x": 24, - "y": 33 + "y": 38 }, - "id": 15, + "id": 18, "title": "Glance service", "type": "row" }, @@ -936,13 +1049,13 @@ "h": 8, "w": 6, "x": 0, - "y": 34 + "y": 39 }, - "id": 16, + "id": 19, "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -977,9 +1090,9 @@ "h": 8, "w": 18, "x": 6, - "y": 34 + "y": 39 }, - "id": 17, + "id": 20, "options": { "legend": { "calcs": [ ], @@ -990,7 +1103,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -1036,9 +1149,9 @@ "h": 8, "w": 24, "x": 0, - "y": 42 + "y": 47 }, - "id": 18, + "id": 21, "pluginVersion": "v10.0.0", "targets": [ { @@ -1141,7 +1254,7 @@ "label": "Loki data source", "name": "loki_datasource", "query": "loki", - "regex": "(?!grafanacloud.+usage-insights|grafanacloud.+alert-state-history).+", + "regex": "", "type": "datasource" } ] diff --git a/assets/oracledb/dashboards/oracledb-overview.json b/assets/oracledb/dashboards/oracledb-overview.json index e2f59e5..911cf4f 100644 --- a/assets/oracledb/dashboards/oracledb-overview.json +++ b/assets/oracledb/dashboards/oracledb-overview.json @@ -318,7 +318,7 @@ "uid": "$loki_datasource" }, "editorMode": "builder", - "expr": "{filename=~\"/.*/.*/diag/rdbms/.*/.*/trace/alert_.*log\",job=~\"$job\", instance=~\"$instance\"}", + "expr": "{job=~\"$job\", instance=~\"$instance\"} |= `` | (filename=~\"/.*/.*/diag/rdbms/.*/.*/trace/alert_.*log\" or log_type=\"oracledb\")", "queryType": "range", "refId": "A" } diff --git a/assets/pgbouncer/dashboards/clusterOverview b/assets/pgbouncer/dashboards/clusterOverview index 8ad1c46..6c5303d 100644 --- a/assets/pgbouncer/dashboards/clusterOverview +++ b/assets/pgbouncer/dashboards/clusterOverview @@ -49,7 +49,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -117,7 +117,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -166,7 +166,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -216,7 +216,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { diff --git a/assets/pgbouncer/dashboards/logs b/assets/pgbouncer/dashboards/logs index 6ef1138..30e342d 100644 --- a/assets/pgbouncer/dashboards/logs +++ b/assets/pgbouncer/dashboards/logs @@ -141,17 +141,19 @@ }, "gridPos": { "h": 6, - "w": 24 + "w": 24, + "x": 0, + "y": 0 }, "id": 1, - "interval": "30s", + "maxDataPoints": 100, "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -181,7 +183,9 @@ }, "gridPos": { "h": 18, - "w": 24 + "w": 24, + "x": 0, + "y": 18 }, "id": 2, "options": { @@ -191,7 +195,7 @@ "showTime": false, "wrapLogMessage": false }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -206,7 +210,7 @@ } ], "refresh": "1m", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "pgbouncer" ], diff --git a/assets/pgbouncer/dashboards/overview b/assets/pgbouncer/dashboards/overview index 3701559..653a958 100644 --- a/assets/pgbouncer/dashboards/overview +++ b/assets/pgbouncer/dashboards/overview @@ -54,7 +54,7 @@ "options": { "graphMode": "none" }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -97,7 +97,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -140,7 +140,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -183,7 +183,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -226,7 +226,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -269,7 +269,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -329,7 +329,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -378,7 +378,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -441,7 +441,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -510,7 +510,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -559,7 +559,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -621,7 +621,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -706,7 +706,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -755,7 +755,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -804,7 +804,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { diff --git a/assets/postgres-exporter/dashboards/postgres-overview.json b/assets/postgres-exporter/dashboards/postgres-overview.json index 9b08fa4..8182422 100644 --- a/assets/postgres-exporter/dashboards/postgres-overview.json +++ b/assets/postgres-exporter/dashboards/postgres-overview.json @@ -25,7 +25,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Postgres Overview", + "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { @@ -77,7 +77,7 @@ { "alias": "fetched", "dsType": "prometheus", - "expr": "sum(irate(pg_stat_database_tup_fetched{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "expr": "sum(irate(pg_stat_database_tup_fetched{datname=~\"$db\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "groupBy": [ { @@ -131,7 +131,7 @@ { "alias": "fetched", "dsType": "prometheus", - "expr": "sum(irate(pg_stat_database_tup_returned{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "expr": "sum(irate(pg_stat_database_tup_returned{datname=~\"$db\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "groupBy": [ { @@ -185,7 +185,7 @@ { "alias": "fetched", "dsType": "prometheus", - "expr": "sum(irate(pg_stat_database_tup_inserted{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "expr": "sum(irate(pg_stat_database_tup_inserted{datname=~\"$db\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "groupBy": [ { @@ -239,7 +239,7 @@ { "alias": "fetched", "dsType": "prometheus", - "expr": "sum(irate(pg_stat_database_tup_updated{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "expr": "sum(irate(pg_stat_database_tup_updated{datname=~\"$db\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "groupBy": [ { @@ -293,7 +293,7 @@ { "alias": "fetched", "dsType": "prometheus", - "expr": "sum(irate(pg_stat_database_tup_deleted{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "expr": "sum(irate(pg_stat_database_tup_deleted{datname=~\"$db\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "groupBy": [ { @@ -396,7 +396,7 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Postgres Overview", + "datasource": "$datasource", "decimals": 0, "editable": true, "error": false, @@ -460,7 +460,7 @@ "targets": [ { "dsType": "prometheus", - "expr": "sum(irate(pg_stat_database_xact_commit{datname=~\"$db\",instance=~\"$instance\"}[5m])) + sum(irate(pg_stat_database_xact_rollback{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "expr": "sum(irate(pg_stat_database_xact_commit{datname=~\"$db\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) + sum(irate(pg_stat_database_xact_rollback{datname=~\"$db\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "groupBy": [ { @@ -530,7 +530,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Postgres Overview", + "datasource": "$datasource", "decimals": 1, "editable": true, "error": false, @@ -584,7 +584,7 @@ { "alias": "Buffers Allocated", "dsType": "prometheus", - "expr": "irate(pg_stat_bgwriter_buffers_alloc_total{instance='$instance'}[5m])", + "expr": "irate(pg_stat_bgwriter_buffers_alloc{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "groupBy": [ { @@ -636,7 +636,7 @@ { "alias": "Buffers Allocated", "dsType": "prometheus", - "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync_total{instance='$instance'}[5m])", + "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "groupBy": [ { @@ -688,7 +688,7 @@ { "alias": "Buffers Allocated", "dsType": "prometheus", - "expr": "irate(pg_stat_bgwriter_buffers_backend_total{instance='$instance'}[5m])", + "expr": "irate(pg_stat_bgwriter_buffers_backend{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "groupBy": [ { @@ -740,7 +740,7 @@ { "alias": "Buffers Allocated", "dsType": "prometheus", - "expr": "irate(pg_stat_bgwriter_buffers_clean_total{instance='$instance'}[5m])", + "expr": "irate(pg_stat_bgwriter_buffers_clean{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "groupBy": [ { @@ -792,7 +792,7 @@ { "alias": "Buffers Allocated", "dsType": "prometheus", - "expr": "irate(pg_stat_bgwriter_buffers_checkpoint_total{instance='$instance'}[5m])", + "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "groupBy": [ { @@ -889,7 +889,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Postgres Overview", + "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { @@ -939,7 +939,7 @@ { "alias": "conflicts", "dsType": "prometheus", - "expr": "sum(rate(pg_stat_database_deadlocks{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "expr": "sum(rate(pg_stat_database_deadlocks{datname=~\"$db\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "groupBy": [ { @@ -991,7 +991,7 @@ { "alias": "deadlocks", "dsType": "prometheus", - "expr": "sum(rate(pg_stat_database_conflicts{datname=~\"$db\",instance=~\"$instance\"}[5m]))", + "expr": "sum(rate(pg_stat_database_conflicts{datname=~\"$db\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "groupBy": [ { @@ -1088,7 +1088,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Postgres Overview", + "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { @@ -1136,10 +1136,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=~\"$instance\"}) / (sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=~\"$instance\"}) + sum(pg_stat_database_blks_read{datname=~\"$db\",instance=~\"$instance\"}))", + "expr": "sum by (datname) (rate(pg_stat_database_blks_hit{datname=~\"$db\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) / (sum by (datname)(rate(pg_stat_database_blks_hit{datname=~\"$db\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) + sum by (datname)(rate(pg_stat_database_blks_read{datname=~\"$db\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "cache hit rate", + "legendFormat": "{{datname}} - cache hit rate", "refId": "A", "step": 240 } @@ -1191,7 +1191,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Postgres Overview", + "datasource": "$datasource", "editable": true, "error": false, "fieldConfig": { @@ -1239,10 +1239,10 @@ "steppedLine": false, "targets": [ { - "expr": "pg_stat_database_numbackends{datname=~\"$db\",instance=~\"$instance\"}", + "expr": "pg_stat_database_numbackends{datname=~\"$db\",job=~\"$job\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{__name__}}", + "legendFormat": "{{datname}} - {{__name__}}", "refId": "A", "step": 240 } @@ -1299,66 +1299,9 @@ "templating": { "list": [ { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "Postgres Overview", - "definition": "", - "hide": 0, - "includeAll": true, - "label": null, - "multi": false, - "name": "instance", - "options": [ ], - "query": "label_values(up{job=~\"postgres.*\"},instance)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "Postgres Overview", - "definition": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", - "hide": 0, - "includeAll": true, - "label": "db", - "multi": false, - "name": "db", - "options": [ ], - "query": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "Postgres Overview", - "value": "Postgres Overview" - }, "hide": 0, "includeAll": false, - "label": "datasource", + "label": "Data Source", "multi": false, "name": "datasource", "options": [ ], @@ -1369,26 +1312,15 @@ "type": "datasource" }, { - "allValue": null, - "current": { - "selected": true, - "text": "postgres", - "value": "postgres" - }, + "allValue": ".+", "datasource": "$datasource", "definition": "label_values(pg_up, job)", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "job", - "multi": false, + "multi": true, "name": "job", - "options": [ - { - "selected": true, - "text": "postgres", - "value": "postgres" - } - ], + "options": [ ], "query": "label_values(pg_up, job)", "refresh": 0, "regex": "", @@ -1399,6 +1331,48 @@ "tagsQuery": "", "type": "query", "useTags": false + }, + { + "allValue": ".+", + "datasource": "$datasource", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [ ], + "query": "label_values(up{job=~\"$job\"},instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "datasource": "$datasource", + "definition": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", + "hide": 0, + "includeAll": true, + "label": "db", + "multi": false, + "name": "db", + "options": [ ], + "query": "label_values(pg_stat_database_tup_fetched{instance=~\"$instance\",datname!~\"template.*|postgres\"},datname)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false } ] }, diff --git a/assets/presto/dashboards/presto-logs.json b/assets/presto/dashboards/presto-logs.json index 8abcdb4..ecdf4e8 100644 --- a/assets/presto/dashboards/presto-logs.json +++ b/assets/presto/dashboards/presto-logs.json @@ -142,17 +142,19 @@ }, "gridPos": { "h": 6, - "w": 24 + "w": 24, + "x": 0, + "y": 0 }, "id": 1, - "interval": "30s", + "maxDataPoints": 100, "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -182,7 +184,9 @@ }, "gridPos": { "h": 18, - "w": 24 + "w": 24, + "x": 0, + "y": 18 }, "id": 2, "options": { @@ -192,7 +196,7 @@ "showTime": false, "wrapLogMessage": true }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -207,7 +211,7 @@ } ], "refresh": "1m", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "presto-mixin" ], diff --git a/assets/prometheus-operator/alerts.yaml b/assets/prometheus-operator/alerts.yaml index e676276..3d1e2f0 100644 --- a/assets/prometheus-operator/alerts.yaml +++ b/assets/prometheus-operator/alerts.yaml @@ -35,12 +35,23 @@ groups: annotations: description: '{{ $value | humanizePercentage }} of reconciling operations failed for {{ $labels.controller }} controller in {{ $labels.namespace }} namespace.' - summary: Errors while reconciling controller. + summary: Errors while reconciling objects. expr: | (sum by (controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="prometheus-operator"}[5m]))) / (sum by (controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="prometheus-operator"}[5m]))) > 0.1 for: 10m labels: severity: warning + - alert: PrometheusOperatorStatusUpdateErrors + annotations: + description: '{{ $value | humanizePercentage }} of status update operations + failed for {{ $labels.controller }} controller in {{ $labels.namespace }} + namespace.' + summary: Errors while updating objects status. + expr: | + (sum by (controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="prometheus-operator"}[5m]))) / (sum by (controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="prometheus-operator"}[5m]))) > 0.1 + for: 10m + labels: + severity: warning - alert: PrometheusOperatorNodeLookupErrors annotations: description: Errors while reconciling Prometheus in {{ $labels.namespace }} diff --git a/assets/prometheus/alerts.yaml b/assets/prometheus/alerts.yaml index 5ea351b..40463db 100644 --- a/assets/prometheus/alerts.yaml +++ b/assets/prometheus/alerts.yaml @@ -22,6 +22,17 @@ groups: for: 20m labels: severity: warning + - alert: PrometheusKubernetesListWatchFailures + annotations: + description: Kubernetes service discovery of Prometheus {{$labels.instance}} + is experiencing {{ printf "%.0f" $value }} failures with LIST/WATCH requests + to the Kubernetes API in the last 5 minutes. + summary: Requests in Kubernetes SD are failing. + expr: | + increase(prometheus_sd_kubernetes_failures_total{job="prometheus"}[5m]) > 0 + for: 15m + labels: + severity: warning - alert: PrometheusNotificationQueueRunningFull annotations: description: Alert notification queue of Prometheus {{$labels.instance}} is @@ -93,7 +104,7 @@ groups: summary: Prometheus is not ingesting samples. expr: | ( - rate(prometheus_tsdb_head_samples_appended_total{job="prometheus"}[5m]) <= 0 + sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="prometheus"}[5m])) <= 0 and ( sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="prometheus"}) > 0 diff --git a/assets/prometheus/dashboards/prometheus-remote-write.json b/assets/prometheus/dashboards/prometheus-remote-write.json index bc9a7cb..304f88e 100644 --- a/assets/prometheus/dashboards/prometheus-remote-write.json +++ b/assets/prometheus/dashboards/prometheus-remote-write.json @@ -54,7 +54,7 @@ "steppedLine": false, "targets": [ { - "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} != 0)\n)\n", + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"} != 0)\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", @@ -135,7 +135,7 @@ "steppedLine": false, "targets": [ { - "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n, 0)\n", + "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])\n, 0)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", @@ -229,7 +229,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n- \n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n", + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]))\n- \n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", @@ -324,7 +324,7 @@ "steppedLine": false, "targets": [ { - "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", @@ -405,7 +405,7 @@ "steppedLine": false, "targets": [ { - "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\"}", + "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", @@ -486,7 +486,7 @@ "steppedLine": false, "targets": [ { - "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\"}", + "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", @@ -567,7 +567,7 @@ "steppedLine": false, "targets": [ { - "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\"}", + "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", @@ -661,7 +661,7 @@ "steppedLine": false, "targets": [ { - "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", @@ -742,7 +742,7 @@ "steppedLine": false, "targets": [ { - "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\"}", + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", @@ -1011,7 +1011,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", @@ -1092,7 +1092,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", @@ -1173,7 +1173,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", @@ -1254,7 +1254,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", diff --git a/assets/prometheus/dashboards/prometheus.json b/assets/prometheus/dashboards/prometheus.json index f102c93..ab442f5 100644 --- a/assets/prometheus/dashboards/prometheus.json +++ b/assets/prometheus/dashboards/prometheus.json @@ -80,6 +80,21 @@ "type": "number", "unit": "s" }, + { + "alias": "Cluster", + "colorMode": null, + "colors": [ ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "cluster", + "thresholds": [ ], + "type": "number", + "unit": "short" + }, { "alias": "Instance", "colorMode": null, @@ -139,18 +154,16 @@ ], "targets": [ { - "expr": "count by (job, instance, version) (prometheus_build_info{job=~\"$job\", instance=~\"$instance\"})", + "expr": "count by (cluster, job, instance, version) (prometheus_build_info{cluster=~\"$cluster\", job=~\"$job\", instance=~\"$instance\"})", "format": "table", "instant": true, - "intervalFactor": 2, "legendFormat": "", "refId": "A" }, { - "expr": "max by (job, instance) (time() - process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "expr": "max by (cluster, job, instance) (time() - process_start_time_seconds{cluster=~\"$cluster\", job=~\"$job\", instance=~\"$instance\"})", "format": "table", "instant": true, - "intervalFactor": 2, "legendFormat": "", "refId": "B" } @@ -236,10 +249,9 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m])) by (scrape_job) * 1e3", + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[5m])) by (cluster, job, scrape_job, instance) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{scrape_job}}", + "legendFormat": "{{cluster}}:{{job}}:{{instance}}:{{scrape_job}}", "legendLink": null } ], @@ -311,10 +323,9 @@ "steppedLine": false, "targets": [ { - "expr": "sum(prometheus_sd_discovered_targets{job=~\"$job\",instance=~\"$instance\"})", + "expr": "sum by (cluster, job, instance) (prometheus_sd_discovered_targets{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"})", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Targets", + "legendFormat": "{{cluster}}:{{job}}:{{instance}}", "legendLink": null } ], @@ -398,10 +409,9 @@ "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_target_interval_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "expr": "rate(prometheus_target_interval_length_seconds_sum{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{interval}} configured", + "legendFormat": "{{cluster}}:{{job}}:{{instance}} {{interval}} configured", "legendLink": null } ], @@ -473,38 +483,33 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total[1m]))", + "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "exceeded body size limit: {{job}}", + "legendFormat": "exceeded body size limit: {{cluster}} {{job}} {{instance}}", "legendLink": null }, { - "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))", + "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_sample_limit_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "exceeded sample limit: {{job}}", + "legendFormat": "exceeded sample limit: {{cluster}} {{job}} {{instance}}", "legendLink": null }, { - "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))", + "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "duplicate timestamp: {{job}}", + "legendFormat": "duplicate timestamp: {{cluster}} {{job}} {{instance}}", "legendLink": null }, { - "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))", + "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_bounds_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "out of bounds: {{job}}", + "legendFormat": "out of bounds: {{cluster}} {{job}} {{instance}}", "legendLink": null }, { - "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))", + "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_order_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "out of order: {{job}}", + "legendFormat": "out of order: {{cluster}} {{job}} {{instance}}", "legendLink": null } ], @@ -576,10 +581,9 @@ "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[5m])", + "expr": "rate(prometheus_tsdb_head_samples_appended_total{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m])", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{job}} {{instance}}", + "legendFormat": "{{cluster}} {{job}} {{instance}}", "legendLink": null } ], @@ -663,10 +667,9 @@ "steppedLine": false, "targets": [ { - "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "expr": "prometheus_tsdb_head_series{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{job}} {{instance}} head series", + "legendFormat": "{{cluster}} {{job}} {{instance}} head series", "legendLink": null } ], @@ -738,10 +741,9 @@ "steppedLine": false, "targets": [ { - "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "expr": "prometheus_tsdb_head_chunks{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{job}} {{instance}} head chunks", + "legendFormat": "{{cluster}} {{job}} {{instance}} head chunks", "legendLink": null } ], @@ -825,10 +827,9 @@ "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_engine_query_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "expr": "rate(prometheus_engine_query_duration_seconds_count{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{job}} {{instance}}", + "legendFormat": "{{cluster}} {{job}} {{instance}}", "legendLink": null } ], @@ -900,9 +901,8 @@ "steppedLine": false, "targets": [ { - "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{slice}}", "legendLink": null } @@ -965,7 +965,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -973,6 +973,30 @@ "regex": "", "type": "datasource" }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "cluster", + "multi": true, + "name": "cluster", + "options": [ ], + "query": "label_values(prometheus_build_info{job=\"prometheus\"}, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": ".+", "current": { @@ -987,7 +1011,7 @@ "multi": true, "name": "job", "options": [ ], - "query": "label_values(prometheus_build_info{job=\"prometheus\"}, job)", + "query": "label_values(prometheus_build_info{cluster=~\"$cluster\"}, job)", "refresh": 1, "regex": "", "sort": 2, @@ -1011,7 +1035,7 @@ "multi": true, "name": "instance", "options": [ ], - "query": "label_values(prometheus_build_info{job=~\"$job\"}, instance)", + "query": "label_values(prometheus_build_info{cluster=~\"$cluster\", job=~\"$job\"}, instance)", "refresh": 1, "regex": "", "sort": 2, diff --git a/assets/promtail/alerts.yaml b/assets/promtail/alerts.yaml index b85ce6b..3d1ad5c 100644 --- a/assets/promtail/alerts.yaml +++ b/assets/promtail/alerts.yaml @@ -3,8 +3,9 @@ groups: rules: - alert: PromtailRequestsErrors annotations: - message: | + description: | {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors. + summary: Promtail request error rate is high. expr: | 100 * sum(rate(promtail_request_duration_seconds_count{status_code=~"5..|failed"}[1m])) by (namespace, job, route, instance) / @@ -15,8 +16,9 @@ groups: severity: critical - alert: PromtailRequestLatency annotations: - message: | + description: | {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency. + summary: Promtail request latency P99 is high. expr: | job_status_code_namespace:promtail_request_duration_seconds:99quantile > 1 for: 15m @@ -24,8 +26,9 @@ groups: severity: critical - alert: PromtailFileMissing annotations: - message: | + description: | {{ $labels.instance }} {{ $labels.job }} {{ $labels.path }} matches the glob but is not being tailed. + summary: Promtail cannot find a file it should be tailing. expr: | promtail_file_bytes_total unless promtail_read_bytes_total for: 15m diff --git a/assets/promtail/dashboards/promtail.json b/assets/promtail/dashboards/promtail.json index b896104..36c1b13 100644 --- a/assets/promtail/dashboards/promtail.json +++ b/assets/promtail/dashboards/promtail.json @@ -222,7 +222,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(promtail_read_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1m]))", + "expr": "sum(rate(promtail_read_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "logs read", @@ -298,7 +298,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(promtail_read_lines_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1m]))", + "expr": "sum(rate(promtail_read_lines_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "lines read", @@ -470,7 +470,7 @@ "steppedLine": false, "targets": [ { - "expr": "job:promtail_request_duration_seconds:99quantile{cluster=~\"$cluster\", job=~\"$namespace/promtail\"}", + "expr": "job:promtail_request_duration_seconds:99quantile{cluster=~\"$cluster\", job=~\"$namespace/promtail.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "p99", @@ -478,7 +478,7 @@ "step": 10 }, { - "expr": "job:promtail_request_duration_seconds:50quantile{cluster=~\"$cluster\", job=~\"$namespace/promtail\"}", + "expr": "job:promtail_request_duration_seconds:50quantile{cluster=~\"$cluster\", job=~\"$namespace/promtail.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "p50", @@ -486,7 +486,7 @@ "step": 10 }, { - "expr": "job:promtail_request_duration_seconds:avg{cluster=~\"$cluster\", job=~\"$namespace/promtail\"}", + "expr": "job:promtail_request_duration_seconds:avg{cluster=~\"$cluster\", job=~\"$namespace/promtail.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", diff --git a/assets/snmp/alerts.yaml b/assets/snmp/alerts.yaml index 2f63bc1..0b731c7 100644 --- a/assets/snmp/alerts.yaml +++ b/assets/snmp/alerts.yaml @@ -19,3 +19,12 @@ groups: for: 5m labels: severity: warning + - alert: SNMPRestarted + annotations: + description: The SNMP target on instance {{$labels.instance}} from job {{$labels.job}} + has restarted. + summary: SNMP target restarted. + expr: sysUpTime < 300 + for: 5s + labels: + severity: info diff --git a/assets/snmp/dashboards/snmp-overview.json b/assets/snmp/dashboards/snmp-overview.json index ba1f721..18c5690 100644 --- a/assets/snmp/dashboards/snmp-overview.json +++ b/assets/snmp/dashboards/snmp-overview.json @@ -720,7 +720,7 @@ "datasource": { "uid": "$prometheus_datasource" }, - "expr": "max by (ifDescr) (ifSpeed{job_snmp=~\"$job\", instance=~\"$instance\", snmp_target=~\"$snmp_target\", ifDescr=~\"$interface\"})", + "expr": "max by (ifDescr) (ifHighSpeed{job_snmp=~\"$job\", instance=~\"$instance\", snmp_target=~\"$snmp_target\", ifDescr=~\"$interface\"})", "format": "table", "instant": true, "intervalFactor": 2, diff --git a/assets/spring-boot/dashboards/spring-boot-dashboard.json b/assets/spring-boot/dashboards/spring-boot-dashboard.json deleted file mode 100644 index 0b31284..0000000 --- a/assets/spring-boot/dashboards/spring-boot-dashboard.json +++ /dev/null @@ -1,4017 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Dashboard for Spring Boot2 Statistics(by micrometer-prometheus).", - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 6756, - "graphTooltip": 0, - "id": 2, - "links": [ ], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 54, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "refId": "A" - } - ], - "title": "Basic Statistics", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 1 - }, - "id": 52, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "process_uptime_seconds{application=~\"$application\", instance=~\"$instance\", job=~\"$job\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 14400 - } - ], - "title": "Uptime", - "type": "stat" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 70 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 6, - "w": 5, - "x": 6, - "y": 1 - }, - "id": 58, - "links": [ ], - "maxDataPoints": 100, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(jvm_memory_used_bytes{application=~\"$application\", instance=~\"$instance\", job=~\"$job\", area=\"heap\"})*100/sum(jvm_memory_max_bytes{application=~\"$application\",instance=~\"$instance\", job=~\"$job\", area=\"heap\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 14400 - } - ], - "title": "Heap Used", - "type": "gauge" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - }, - { - "options": { - "from": -100000000000000005366162204393472, - "result": { - "text": "N/A" - }, - "to": 0 - }, - "type": "range" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 70 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 6, - "w": 5, - "x": 11, - "y": 1 - }, - "id": 60, - "links": [ ], - "maxDataPoints": 100, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(jvm_memory_used_bytes{application=~\"$application\", instance=~\"$instance\", job=~\"$job\", area=\"nonheap\"})*100/sum(jvm_memory_max_bytes{application=~\"$application\",instance=~\"$instance\", job=~\"$job\", area=\"nonheap\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 14400 - } - ], - "title": "Non-Heap Used", - "type": "gauge" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "locale" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 1 - }, - "id": 66, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "process_files_open_files{application=~\"$application\", instance=~\"$instance\", job=~\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Open Files", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "process_files_max_files{application=~\"$application\", instance=~\"$instance\", job=~\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Max Files", - "refId": "B" - } - ], - "title": "Process Open Files", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "dateTimeAsIso" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 4 - }, - "id": 56, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "process_start_time_seconds{application=~\"$application\", instance=~\"$instance\", job=~\"$job\"}*1000", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 14400 - } - ], - "title": "Start time", - "type": "stat" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 7 - }, - "id": 95, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "system_cpu_usage{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "System CPU Usage", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "process_cpu_usage{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Process CPU Usage", - "refId": "B" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 7 - }, - "id": 96, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "system_load_average_1m{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Load Average [1m]", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "system_cpu_count{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "CPU Core Size", - "refId": "B" - } - ], - "title": "Load Average", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 48, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "refId": "A" - } - ], - "title": "JVM Statistics - Memory", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 15 - }, - "id": 85, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "repeat": "memory_pool_heap", - "repeatDirection": "h", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "jvm_memory_used_bytes{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", id=~\"$memory_pool_heap\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Used", - "refId": "C" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "jvm_memory_committed_bytes{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", id=~\"$memory_pool_heap\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Commited", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "jvm_memory_max_bytes{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", id=~\"$memory_pool_heap\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Max", - "refId": "B" - } - ], - "title": "$memory_pool_heap (heap)", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 23 - }, - "id": 88, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "repeat": "memory_pool_nonheap", - "repeatDirection": "h", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "jvm_memory_used_bytes{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", id=~\"$memory_pool_nonheap\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Used", - "refId": "C" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "jvm_memory_committed_bytes{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", id=~\"$memory_pool_nonheap\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Commited", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "jvm_memory_max_bytes{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", id=~\"$memory_pool_nonheap\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Max", - "refId": "B" - } - ], - "title": "$memory_pool_nonheap (non-heap)", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "locale" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 31 - }, - "id": 50, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "jvm_classes_loaded_classes{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Classes Loaded", - "refId": "A" - } - ], - "title": "Classes Loaded", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 80, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "irate(jvm_classes_unloaded_classes_total{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Classes Unloaded", - "refId": "A" - } - ], - "title": "Classes Unloaded", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 82, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "jvm_buffer_memory_used_bytes{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", id=\"direct\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Used Bytes", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "jvm_buffer_total_capacity_bytes{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", id=\"direct\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Capacity Bytes", - "refId": "B" - } - ], - "title": "Direct Buffers", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 83, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "jvm_buffer_memory_used_bytes{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", id=\"mapped\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Used Bytes", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "jvm_buffer_total_capacity_bytes{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", id=\"mapped\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Capacity Bytes", - "refId": "B" - } - ], - "title": "Mapped Buffers", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 46 - }, - "id": 68, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "jvm_threads_daemon_threads{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Daemon", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "jvm_threads_live_threads{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Live", - "refId": "B" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "jvm_threads_peak_threads{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Peak", - "refId": "C" - } - ], - "title": "Threads", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 46 - }, - "id": 78, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "irate(jvm_gc_memory_allocated_bytes_total{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "allocated", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "irate(jvm_gc_memory_promoted_bytes_total{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "promoted", - "refId": "B" - } - ], - "title": "Memory Allocate/Promote", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 54 - }, - "id": 72, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "refId": "A" - } - ], - "title": "JVM Statistics - GC", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "locale" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsZero", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - }, - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 55 - }, - "id": 74, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "min", - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "irate(jvm_gc_pause_seconds_count{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{action}} [{{cause}}]", - "refId": "A" - } - ], - "title": "GC Count", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsZero", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - }, - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 55 - }, - "id": 76, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "min", - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "irate(jvm_gc_pause_seconds_sum{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{action}} [{{cause}}]", - "refId": "A" - } - ], - "title": "GC Stop the World Duration", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 65 - }, - "id": 34, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "refId": "A" - } - ], - "title": "HikariCP Statistics", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 66 - }, - "id": 44, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "hikaricp_connections{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", pool=\"$hikaricp\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Connections Size", - "type": "stat" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 20, - "x": 4, - "y": 66 - }, - "id": 36, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "hikaricp_connections_active{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", pool=\"$hikaricp\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Active", - "refId": "B" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "hikaricp_connections_idle{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", pool=\"$hikaricp\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Idle", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "hikaricp_connections_pending{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", pool=\"$hikaricp\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pending", - "refId": "C" - } - ], - "title": "Connections", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 70 - }, - "id": 46, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "hikaricp_connections_timeout_total{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", pool=\"$hikaricp\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Connection Timeout Count", - "type": "stat" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 74 - }, - "id": 38, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "hikaricp_connections_creation_seconds_sum{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", pool=\"$hikaricp\"} / hikaricp_connections_creation_seconds_count{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", pool=\"$hikaricp\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Creation Time", - "refId": "A" - } - ], - "title": "Connection Creation Time", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 74 - }, - "id": 42, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "hikaricp_connections_usage_seconds_sum{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", pool=\"$hikaricp\"} / hikaricp_connections_usage_seconds_count{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", pool=\"$hikaricp\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Usage Time", - "refId": "A" - } - ], - "title": "Connection Usage Time", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 74 - }, - "id": 40, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "hikaricp_connections_acquire_seconds_sum{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", pool=\"$hikaricp\"} / hikaricp_connections_acquire_seconds_count{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", pool=\"$hikaricp\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Acquire Time", - "refId": "A" - } - ], - "title": "Connection Acquire Time", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 80 - }, - "id": 18, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "refId": "A" - } - ], - "title": "HTTP Statistics", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 81 - }, - "id": 4, - "links": [ ], - "options": { - "legend": { - "calcs": [ ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "irate(http_server_requests_seconds_count{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", uri!~\".*actuator.*\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{method}} [{{status}}] - {{uri}}", - "refId": "A" - } - ], - "title": "Request Count", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 88 - }, - "id": 2, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "min" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "irate(http_server_requests_seconds_sum{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", exception=~\"[n|N]one\", uri!~\".*actuator.*\"}[$__rate_interval]) / irate(http_server_requests_seconds_count{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", exception=~\"[n|N]one\", uri!~\".*actuator.*\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{method}} [{{status}}] - {{uri}}", - "refId": "A" - } - ], - "title": "Response Time", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 95 - }, - "id": 22, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "refId": "A" - } - ], - "title": "Tomcat Statistics", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "locale" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 96 - }, - "id": 28, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "tomcat_global_error_total{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Error Count", - "type": "stat" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 9, - "x": 4, - "y": 96 - }, - "id": 24, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "tomcat_sessions_active_current_sessions{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "active sessions", - "refId": "A" - } - ], - "title": "Active Sessions", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 11, - "x": 13, - "y": 96 - }, - "id": 26, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "irate(tomcat_global_sent_bytes_total{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Sent Bytes", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "irate(tomcat_global_received_bytes_total{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Recieved Bytes", - "refId": "B" - } - ], - "title": "Sent & Recieved Bytes", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "locale" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 100 - }, - "id": 32, - "links": [ ], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "tomcat_threads_config_max_threads{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Thread Config Max", - "type": "stat" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 103 - }, - "id": 30, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "tomcat_threads_current_threads{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Current thread", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "tomcat_threads_busy{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Current thread busy", - "refId": "B" - } - ], - "title": "Threads", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 110 - }, - "id": 8, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "grafanacloud-prom" - }, - "refId": "A" - } - ], - "title": "Logback Statistics", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 111 - }, - "id": 6, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min", - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "alias": "", - "datasource": { - "uid": "$datasource" - }, - "expr": "irate(logback_events_total{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", level=\"info\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "info", - "rawSql": "SELECT\n $__time(time_column),\n value1\nFROM\n metric_table\nWHERE\n $__timeFilter(time_column)\n", - "refId": "A" - } - ], - "title": "INFO logs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 111 - }, - "id": 10, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min", - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "alias": "", - "datasource": { - "uid": "$datasource" - }, - "expr": "irate(logback_events_total{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", level=\"error\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "error", - "rawSql": "SELECT\n $__time(time_column),\n value1\nFROM\n metric_table\nWHERE\n $__timeFilter(time_column)\n", - "refId": "A" - } - ], - "title": "ERROR logs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 118 - }, - "id": 14, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min", - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "alias": "", - "datasource": { - "uid": "$datasource" - }, - "expr": "irate(logback_events_total{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", level=\"warn\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "warn", - "rawSql": "SELECT\n $__time(time_column),\n value1\nFROM\n metric_table\nWHERE\n $__timeFilter(time_column)\n", - "refId": "A" - } - ], - "title": "WARN logs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 118 - }, - "id": 16, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min", - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "alias": "", - "datasource": { - "uid": "$datasource" - }, - "expr": "irate(logback_events_total{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", level=\"debug\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "debug", - "rawSql": "SELECT\n $__time(time_column),\n value1\nFROM\n metric_table\nWHERE\n $__timeFilter(time_column)\n", - "refId": "A" - } - ], - "title": "DEBUG logs", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 118 - }, - "id": 20, - "links": [ ], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min", - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "alias": "", - "datasource": { - "uid": "$datasource" - }, - "expr": "irate(logback_events_total{instance=~\"$instance\", job=~\"$job\", application=~\"$application\", level=\"trace\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "trace", - "rawSql": "SELECT\n $__time(time_column),\n value1\nFROM\n metric_table\nWHERE\n $__timeFilter(time_column)\n", - "refId": "A" - } - ], - "title": "TRACE logs", - "type": "timeseries" - } - ], - "refresh": "30s", - "schemaVersion": 38, - "style": "dark", - "tags": [ ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "label": "Data source", - "multi": false, - "name": "datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "(?!grafanacloud-usage|grafanacloud-ml-metrics).+", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "$datasource" - }, - "definition": "label_values(jvm_classes_loaded_classes, job)", - "hide": 0, - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "options": [ ], - "query": { - "query": "label_values(jvm_classes_loaded_classes, job)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "$datasource" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Instance", - "multi": true, - "name": "instance", - "options": [ ], - "query": "label_values(jvm_classes_loaded_classes{job=~\"$job\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "$datasource" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Application", - "multi": true, - "name": "application", - "options": [ ], - "query": "label_values(jvm_classes_loaded_classes{instance=~\"$instance\", job=~\"$job\"}, application)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "$datasource" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "HikariCP-Pool", - "multi": true, - "name": "hikaricp", - "options": [ ], - "query": "label_values(hikaricp_connections{instance=~\"$instance\", job=~\"$job\", application=~\"$application\"}, pool)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "$datasource" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Memory Pool (heap)", - "multi": true, - "name": "memory_pool_heap", - "options": [ ], - "query": "label_values(jvm_memory_used_bytes{application=~\"$application\", instance=~\"$instance\", job=~\"$job\", area=\"heap\"},id)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "uid": "$datasource" - }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Memory Pool (nonheap)", - "multi": true, - "name": "memory_pool_nonheap", - "options": [ ], - "query": "label_values(jvm_memory_used_bytes{application=~\"$application\", instance=~\"$instance\", job=~\"$job\", area=\"nonheap\"},id)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Spring Boot Statistics", - "uid": "20201230-spring", - "version": 3, - "weekStart": "" -} diff --git a/assets/spring-boot/rules.yaml b/assets/spring-boot/rules.yaml index 19765bd..0967ef4 100644 --- a/assets/spring-boot/rules.yaml +++ b/assets/spring-boot/rules.yaml @@ -1 +1 @@ -null +{} diff --git a/assets/squid/dashboards/squid-overview.json b/assets/squid/dashboards/squid-overview.json index 02f3d4a..f0439b5 100644 --- a/assets/squid/dashboards/squid-overview.json +++ b/assets/squid/dashboards/squid-overview.json @@ -1590,7 +1590,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{filename=\"/var/log/squid/cache.log\", job=~\"$job\", instance=~\"$instance\"} |= ``", + "expr": "{job=~\"$job\", instance=~\"$instance\"} |= `` | (filename=\"/var/log/squid/cache.log\" or log_type=\"cache\")", "queryType": "range", "refId": "A" } @@ -1626,7 +1626,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{filename=\"/var/log/squid/access.log\", job=~\"$job\", instance=~\"$instance\"} |= ``", + "expr": "{job=~\"$job\", instance=~\"$instance\"} |= `` | (filename=\"/var/log/squid/access.log\" or log_type=\"access\")", "queryType": "range", "refId": "A" } @@ -1688,6 +1688,28 @@ "type": "query", "useTags": false }, + { + "allValue": ".*", + "current": { }, + "datasource": { + "uid": "${prometheus_datasource}" + }, + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [ ], + "query": "label_values(squid_server_http_requests_total{job=~\"$job\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": "", "current": { }, @@ -1700,7 +1722,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(squid_server_http_requests_total{job=~\"$job\"},instance)", + "query": "label_values(squid_server_http_requests_total{job=~\"$job\"}, instance)", "refresh": 2, "regex": "", "sort": 0, diff --git a/assets/thanos/alerts.yaml b/assets/thanos/alerts.yaml index 7a0106e..6c24ee4 100644 --- a/assets/thanos/alerts.yaml +++ b/assets/thanos/alerts.yaml @@ -173,9 +173,9 @@ groups: - alert: ThanosQueryOverload annotations: description: Thanos Query {{$labels.job}} has been overloaded for more than - 15 minutes. This may be a symptom of excessive simultanous complex requests, + 15 minutes. This may be a symptom of excessive simultaneous complex requests, low performance of the Prometheus API, or failures within these components. - Assess the health of the Thanos query instances, the connnected Prometheus + Assess the health of the Thanos query instances, the connected Prometheus instances, look for potential senders of these requests and then contact support. runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosqueryoverload summary: Thanos query reaches its maximum capacity serving concurrent requests. diff --git a/assets/thanos/dashboards/receive.json b/assets/thanos/dashboards/receive.json index 39246cb..cb9caa4 100644 --- a/assets/thanos/dashboards/receive.json +++ b/assets/thanos/dashboards/receive.json @@ -1365,7 +1365,7 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", - "description": "Shows ratio of errors compared to the total number of forwareded requests to other receive nodes.", + "description": "Shows ratio of errors compared to the total number of forwarded requests to other receive nodes.", "fill": 10, "id": 17, "legend": { diff --git a/assets/thanos/rules.yaml b/assets/thanos/rules.yaml index f18cf5b..48e86d7 100644 --- a/assets/thanos/rules.yaml +++ b/assets/thanos/rules.yaml @@ -44,14 +44,14 @@ groups: / sum by (job) (rate(grpc_server_started_total{job=~".*thanos-receive.*", grpc_type="unary"}[5m])) ) - record: :grpc_server_failures_per_unary:sum_rate + record: :thanos_query_receive_grpc_server_failures_per_unary:sum_rate - expr: | ( sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~".*thanos-receive.*", grpc_type="server_stream"}[5m])) / sum by (job) (rate(grpc_server_started_total{job=~".*thanos-receive.*", grpc_type="server_stream"}[5m])) ) - record: :grpc_server_failures_per_stream:sum_rate + record: :thanos_query_receive_grpc_server_failures_per_stream:sum_rate - expr: | ( sum by (job) (rate(http_requests_total{handler="receive", job=~".*thanos-receive.*", code!~"5.."}[5m])) @@ -95,14 +95,14 @@ groups: / sum by (job) (rate(grpc_server_started_total{job=~".*thanos-store.*", grpc_type="unary"}[5m])) ) - record: :grpc_server_failures_per_unary:sum_rate + record: :thanos_query_store_grpc_server_failures_per_unary:sum_rate - expr: | ( sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~".*thanos-store.*", grpc_type="server_stream"}[5m])) / sum by (job) (rate(grpc_server_started_total{job=~".*thanos-store.*", grpc_type="server_stream"}[5m])) ) - record: :grpc_server_failures_per_stream:sum_rate + record: :thanos_query_store_grpc_server_failures_per_stream:sum_rate - expr: | ( sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{job=~".*thanos-store.*"}[5m])) diff --git a/assets/varnish/dashboards/varnish-overview.json b/assets/varnish/dashboards/varnish-overview.json index d8241be..7be525e 100644 --- a/assets/varnish/dashboards/varnish-overview.json +++ b/assets/varnish/dashboards/varnish-overview.json @@ -73,7 +73,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "avg((rate(varnish_main_cache_hit{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) / clamp_min(rate(varnish_main_cache_hit{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) + rate(varnish_main_cache_miss{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]), 1))) * 100\n", + "expr": "avg((rate(varnish_main_cache_hit{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / clamp_min(rate(varnish_main_cache_hit{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) + rate(varnish_main_cache_miss{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]), 1))) * 100\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}" @@ -317,7 +317,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_cache_hit{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_main_cache_hit{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}" @@ -378,7 +378,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_cache_hitpass{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_main_cache_hitpass{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}" @@ -439,7 +439,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "varnish_main_thread_queue_len{job=~\"$job\",instance=~\"$instance\"}", + "expr": "varnish_main_thread_queue_len{job=~\"$job\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}" @@ -500,7 +500,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "varnish_main_pools{job=~\"$job\",instance=~\"$instance\"}", + "expr": "varnish_main_pools{job=~\"$job\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}" @@ -591,7 +591,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_backend_conn{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_main_backend_conn{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Accepted" @@ -600,7 +600,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_backend_recycle{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_main_backend_recycle{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Recycled" @@ -609,7 +609,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_backend_reuse{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_main_backend_reuse{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Reused" @@ -618,7 +618,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_backend_busy{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_main_backend_busy{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Busy" @@ -627,7 +627,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_backend_unhealthy{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_main_backend_unhealthy{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Unhealthy" @@ -714,7 +714,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_sessions{job=~\"$job\",instance=~\"$instance\",type=\"conn\"}[$__rate_interval])", + "expr": "irate(varnish_main_sessions{job=~\"$job\", instance=~\"$instance\",type=\"conn\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Connected" @@ -723,7 +723,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_sessions{job=~\"$job\",instance=~\"$instance\",type=\"queued\"}[$__rate_interval])", + "expr": "irate(varnish_main_sessions{job=~\"$job\", instance=~\"$instance\",type=\"queued\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Queued" @@ -732,7 +732,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_sessions{job=~\"$job\",instance=~\"$instance\",type=\"dropped\"}[$__rate_interval])", + "expr": "irate(varnish_main_sessions{job=~\"$job\", instance=~\"$instance\",type=\"dropped\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Dropped" @@ -819,7 +819,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_client_req{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_main_client_req{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Frontend" @@ -828,7 +828,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_backend_req{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_main_backend_req{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Backend" @@ -928,7 +928,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "avg by (instance, job) ((rate(varnish_main_cache_hit{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) / clamp_min(rate(varnish_main_cache_hit{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) + rate(varnish_main_cache_miss{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]), 1))) * 100", + "expr": "avg by (instance, job) ((rate(varnish_main_cache_hit{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) / clamp_min(rate(varnish_main_cache_hit{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) + rate(varnish_main_cache_miss{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]), 1))) * 100", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}" @@ -1019,7 +1019,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "varnish_sma_g_bytes{job=~\"$job\",instance=~\"$instance\",type=\"s0\"}", + "expr": "varnish_sma_g_bytes{job=~\"$job\", instance=~\"$instance\",type=\"s0\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}" @@ -1110,7 +1110,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_n_expired{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_main_n_expired{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Expired" @@ -1119,7 +1119,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_n_lru_nuked{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_main_n_lru_nuked{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Nuked" @@ -1206,7 +1206,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_s_resp_hdrbytes{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_main_s_resp_hdrbytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Frontend header" @@ -1215,7 +1215,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_main_s_resp_bodybytes{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_main_s_resp_bodybytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Frontend body" @@ -1224,7 +1224,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_backend_beresp_hdrbytes{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_backend_beresp_hdrbytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{backend}} - Backend header" @@ -1233,7 +1233,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "irate(varnish_backend_beresp_bodybytes{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "irate(varnish_backend_beresp_bodybytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{backend}} - Backend body" @@ -1357,7 +1357,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "increase(varnish_main_threads_failed{job=~\"$job\",instance=~\"$instance\"}[$__interval:])", + "expr": "increase(varnish_main_threads_failed{job=~\"$job\", instance=~\"$instance\"}[$__interval:])", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1367,7 +1367,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "increase(varnish_main_threads_created{job=~\"$job\",instance=~\"$instance\"}[$__interval:])", + "expr": "increase(varnish_main_threads_created{job=~\"$job\", instance=~\"$instance\"}[$__interval:])", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1377,7 +1377,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "increase(varnish_main_threads_limited{job=~\"$job\",instance=~\"$instance\"}[$__interval:])", + "expr": "increase(varnish_main_threads_limited{job=~\"$job\", instance=~\"$instance\"}[$__interval:])", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1387,7 +1387,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "varnish_main_threads{job=~\"$job\",instance=~\"$instance\"}", + "expr": "varnish_main_threads{job=~\"$job\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - Total" @@ -1440,7 +1440,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{filename=~\"/var/log/varnish/varnishncsa-frontend.*.log|/opt/varnish/log/varnishncsa-frontend.*.log\", job=~\"$job\", instance=~\"$instance\"}", + "expr": "{job=~\"$job\", instance=~\"$instance\", filename=~\"/var/log/varnish/varnishncsa-frontend.*.log|/opt/varnish/log/varnishncsa-frontend.*.log\"}", "queryType": "range", "refId": "A" } @@ -1476,7 +1476,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{filename=~\"/var/log/varnish/varnishncsa-backend.*.log|/opt/varnish/log/varnishncsa-backend.*.log\", job=~\"$job\", instance=~\"$instance\"}", + "expr": "{job=~\"$job\", instance=~\"$instance\", filename=~\"/var/log/varnish/varnishncsa-backend.*.log|/opt/varnish/log/varnishncsa-backend.*.log\"}", "queryType": "range", "refId": "A" } @@ -1517,7 +1517,7 @@ "type": "datasource" }, { - "allValue": "", + "allValue": ".+", "current": { }, "datasource": { "uid": "${prometheus_datasource}" @@ -1539,7 +1539,29 @@ "useTags": false }, { - "allValue": "", + "allValue": ".*", + "current": { }, + "datasource": { + "uid": "${prometheus_datasource}" + }, + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [ ], + "query": "label_values(varnish_main_sessions{job=~\"$job\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", "current": { }, "datasource": { "uid": "${prometheus_datasource}" @@ -1550,7 +1572,7 @@ "multi": true, "name": "instance", "options": [ ], - "query": "label_values(varnish_main_sessions,instance)", + "query": "label_values(varnish_main_sessions{job=~\"$job\"},instance)", "refresh": 2, "regex": "", "sort": 0, diff --git a/assets/velero/dashboards/clusterOverview b/assets/velero/dashboards/clusterOverview index 173bbc8..9745d7d 100644 --- a/assets/velero/dashboards/clusterOverview +++ b/assets/velero/dashboards/clusterOverview @@ -44,7 +44,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -88,7 +88,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -132,7 +132,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -176,7 +176,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -244,7 +244,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -308,7 +308,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -374,7 +374,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -422,7 +422,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -486,7 +486,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { diff --git a/assets/velero/dashboards/logs b/assets/velero/dashboards/logs index 2ac6aab..5c43421 100644 --- a/assets/velero/dashboards/logs +++ b/assets/velero/dashboards/logs @@ -141,17 +141,19 @@ }, "gridPos": { "h": 6, - "w": 24 + "w": 24, + "x": 0, + "y": 0 }, "id": 1, - "interval": "30s", + "maxDataPoints": 100, "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -181,7 +183,9 @@ }, "gridPos": { "h": 18, - "w": 24 + "w": 24, + "x": 0, + "y": 18 }, "id": 2, "options": { @@ -191,7 +195,7 @@ "showTime": false, "wrapLogMessage": false }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -206,7 +210,7 @@ } ], "refresh": "1m", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "velero" ], diff --git a/assets/velero/dashboards/overview b/assets/velero/dashboards/overview index fde360b..9ad11fe 100644 --- a/assets/velero/dashboards/overview +++ b/assets/velero/dashboards/overview @@ -132,7 +132,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -176,7 +176,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -220,7 +220,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -264,7 +264,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -325,7 +325,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -392,7 +392,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -441,7 +441,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -535,7 +535,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -602,7 +602,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -663,7 +663,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -730,7 +730,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -791,7 +791,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -858,7 +858,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { diff --git a/assets/wildfly/dashboards/wildfly-datasource.json b/assets/wildfly/dashboards/wildfly-datasource.json index 67bbac0..46fc357 100644 --- a/assets/wildfly/dashboards/wildfly-datasource.json +++ b/assets/wildfly/dashboards/wildfly-datasource.json @@ -521,6 +521,28 @@ "type": "query", "useTags": false }, + { + "allValue": ".*", + "current": { }, + "datasource": { + "uid": "${prometheus_datasource}" + }, + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [ ], + "query": "label_values(wildfly_batch_jberet_active_count{job=~\"$job\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": "", "current": { }, @@ -555,7 +577,7 @@ "multi": true, "name": "datasource", "options": [ ], - "query": "label_values(wildfly_datasources_pool_idle_count{}, data_source)", + "query": "label_values(wildfly_datasources_pool_idle_count{job=~\"$job\"}, data_source)", "refresh": 2, "regex": "", "sort": 0, diff --git a/assets/wildfly/dashboards/wildfly-overview.json b/assets/wildfly/dashboards/wildfly-overview.json index ae93f57..df9e4b8 100644 --- a/assets/wildfly/dashboards/wildfly-overview.json +++ b/assets/wildfly/dashboards/wildfly-overview.json @@ -108,7 +108,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "rate(wildfly_undertow_request_count_total{server=~\"$server\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "rate(wildfly_undertow_request_count_total{server=~\"$server\",job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{server}} - {{http_listener}}{{https_listener}}" @@ -199,7 +199,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "rate(wildfly_undertow_error_count_total{server=~\"$server\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "rate(wildfly_undertow_error_count_total{server=~\"$server\",job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{server}} - {{http_listener}}{{https_listener}}" @@ -290,7 +290,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "rate(wildfly_undertow_bytes_received_total_bytes{server=~\"$server\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "rate(wildfly_undertow_bytes_received_total_bytes{server=~\"$server\",job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{server}} - {{http_listener}}{{https_listener}}" @@ -381,7 +381,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "rate(wildfly_undertow_bytes_sent_total_bytes{server=~\"$server\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "rate(wildfly_undertow_bytes_sent_total_bytes{server=~\"$server\",job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{server}} - {{http_listener}}{{https_listener}}" @@ -419,7 +419,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{filename=\"/opt/wildfly/standalone/log/server.log\", job=~\"$job\",instance=~\"$instance\"}", + "expr": "{job=~\"$job\", instance=~\"$instance\"} |= `` | (filename=~\"/opt/wildfly/.*/server.log\" or log_type=\"wildfly\")", "queryType": "range", "refId": "A" } @@ -524,7 +524,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "wildfly_undertow_active_sessions{deployment=~\"$deployment\", job=~\"$job\", instance=~\"$instance\"}", + "expr": "wildfly_undertow_active_sessions{deployment=~\"$deployment\",job=~\"$job\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{deployment}}" @@ -615,7 +615,7 @@ "datasource": { "uid": "${prometheus_datasource}" }, - "expr": "increase(wildfly_undertow_expired_sessions_total{deployment=~\"$deployment\", job=~\"$job\", instance=~\"$instance\"}[$__interval])", + "expr": "increase(wildfly_undertow_expired_sessions_total{deployment=~\"$deployment\",job=~\"$job\", instance=~\"$instance\"}[$__interval])", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -771,6 +771,28 @@ "type": "query", "useTags": false }, + { + "allValue": ".*", + "current": { }, + "datasource": { + "uid": "${prometheus_datasource}" + }, + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [ ], + "query": "label_values(wildfly_batch_jberet_active_count{job=~\"$job\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": "", "current": { }, @@ -805,7 +827,7 @@ "multi": false, "name": "server", "options": [ ], - "query": "label_values(wildfly_undertow_request_count_total{}, server)", + "query": "label_values(wildfly_undertow_request_count_total{job=~\"$job\"}, server)", "refresh": 2, "regex": "", "sort": 0, @@ -827,7 +849,7 @@ "multi": false, "name": "deployment", "options": [ ], - "query": "label_values(wildfly_undertow_active_sessions{}, deployment)", + "query": "label_values(wildfly_undertow_active_sessions{job=~\"$job\"}, deployment)", "refresh": 2, "regex": "", "sort": 0, diff --git a/assets/windows-active-directory/alerts.yaml b/assets/windows-active-directory/alerts.yaml index e5b49f5..24ae470 100644 --- a/assets/windows-active-directory/alerts.yaml +++ b/assets/windows-active-directory/alerts.yaml @@ -9,6 +9,7 @@ groups: expr: | 100 - (avg without (mode, core) (rate(windows_cpu_time_total{job=~"integrations/windows_exporter", mode="idle"}[2m])) * 100) > 90 for: 15m + keep_firing_for: 5m labels: severity: warning - alert: WindowsMemoryHighUtilization @@ -21,6 +22,7 @@ groups: / windows_cs_physical_memory_bytes{job=~"integrations/windows_exporter"}) * 100) > 90 for: 15m + keep_firing_for: 5m labels: severity: critical - alert: WindowsDiskAlmostOutOfSpace @@ -31,6 +33,7 @@ groups: expr: | 100 - ((windows_logical_disk_free_bytes{job=~"integrations/windows_exporter"} ) / (windows_logical_disk_size_bytes{job=~"integrations/windows_exporter"})) * 100 > 90 for: 15m + keep_firing_for: 5m labels: severity: critical - alert: WindowsServiceNotHealthy @@ -56,21 +59,23 @@ groups: - alert: WindowsNTPClientDelay annotations: description: | - 'Round-trip time of NTP client on instance {{ $labels.instance }} is greater than 1 second. Delay is {{ $value }} sec.' + Round-trip time of NTP client on instance {{ $labels.instance }} is greater than 1 second. Delay is {{ $value }} sec. summary: NTP client delay. expr: | windows_time_ntp_round_trip_delay_seconds{job=~"integrations/windows_exporter"} > 1 for: 5m + keep_firing_for: 5m labels: severity: warning - alert: WindowsNTPTimeOffset annotations: description: | - 'NTP time offset for instance {{ $labels.instance }} is greater than 1 second. Offset is {{ $value }} sec.' + NTP time offset for instance {{ $labels.instance }} is greater than 1 second. Offset is {{ $value }} sec. summary: NTP time offset is too large. expr: | windows_time_computed_time_offset_seconds{job=~"integrations/windows_exporter"} > 1 for: 5m + keep_firing_for: 5m labels: severity: warning - alert: WindowsActiveDirectoryHighPendingReplicationOperations @@ -83,6 +88,7 @@ groups: expr: "windows_ad_replication_pending_operations{job=~\"integrations/windows_exporter\"} >= 50 \n" for: 10m + keep_firing_for: 5m labels: severity: warning - alert: WindowsActiveDirectoryHighReplicationSyncRequestFailures @@ -95,18 +101,21 @@ groups: expr: "increase(windows_ad_replication_sync_requests_schema_mismatch_failure_total{job=~\"integrations/windows_exporter\"}[5m]) > 0 \n" for: 5m + keep_firing_for: 5m labels: severity: critical - alert: WindowsActiveDirectoryHighPasswordChanges annotations: description: The number of password changes on {{$labels.instance}} is {{ printf - "%.0f" $value }} which is greater than the threshold of 25 + "%.0f" $value }} which is greater than the threshold of 25. This alert would + resolve itself if no new anomalies are detected within 24 hours. summary: There is a high number of password changes. This may indicate unauthorized changes or attacks. expr: | increase(windows_ad_sam_password_changes_total{job=~"integrations/windows_exporter"}[5m]) > 25 for: 5m labels: + keep_firing_for: 24h severity: warning - alert: WindowsActiveDirectoryMetricsDown annotations: diff --git a/assets/windows-active-directory/dashboards/activedirectory b/assets/windows-active-directory/dashboards/activedirectory index c40dd7b..d3a0346 100644 --- a/assets/windows-active-directory/dashboards/activedirectory +++ b/assets/windows-active-directory/dashboards/activedirectory @@ -88,7 +88,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -132,7 +132,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -195,7 +195,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -244,7 +244,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -293,7 +293,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -558,7 +558,7 @@ "y": 15 }, "id": 7, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -714,7 +714,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -769,7 +769,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -822,7 +822,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -965,7 +965,7 @@ "y": 33 }, "id": 11, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -1076,7 +1076,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -1102,11 +1102,10 @@ "label": "Data source", "name": "datasource", "query": "prometheus", - "regex": "Prometheus|Cortex|Mimir|grafanacloud-.+-prom", + "regex": "", "type": "datasource" }, { - "allValue": ".+", "datasource": { "type": "prometheus", "uid": "${datasource}" @@ -1115,13 +1114,12 @@ "label": "Job", "multi": true, "name": "job", - "query": "label_values(windows_os_info{job=~\"integrations/windows_exporter\"}, job)", + "query": "label_values(windows_ad_directory_service_threads{job=~\"integrations/windows_exporter\"}, job)", "refresh": 2, "sort": 1, "type": "query" }, { - "allValue": ".+", "datasource": { "type": "prometheus", "uid": "${datasource}" @@ -1130,7 +1128,7 @@ "label": "Instance", "multi": true, "name": "instance", - "query": "label_values(windows_os_info{job=~\"integrations/windows_exporter\",job=~\"$job\"}, instance)", + "query": "label_values(windows_ad_directory_service_threads{job=~\"integrations/windows_exporter\",job=~\"$job\"}, instance)", "refresh": 2, "sort": 1, "type": "query" @@ -1140,7 +1138,7 @@ "label": "Loki data source", "name": "loki_datasource", "query": "loki", - "regex": "Loki|.+logs", + "regex": "", "type": "datasource" } ] diff --git a/assets/windows-active-directory/dashboards/logs b/assets/windows-active-directory/dashboards/logs index db838bc..f5ff4d6 100644 --- a/assets/windows-active-directory/dashboards/logs +++ b/assets/windows-active-directory/dashboards/logs @@ -185,17 +185,19 @@ }, "gridPos": { "h": 6, - "w": 24 + "w": 24, + "x": 0, + "y": 0 }, "id": 1, - "interval": "30s", + "maxDataPoints": 100, "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -225,7 +227,9 @@ }, "gridPos": { "h": 18, - "w": 24 + "w": 24, + "x": 0, + "y": 18 }, "id": 2, "options": { @@ -235,7 +239,7 @@ "showTime": false, "wrapLogMessage": false }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -250,7 +254,7 @@ } ], "refresh": "1m", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "active-directory" ], diff --git a/assets/windows/alerts.yaml b/assets/windows/alerts.yaml index b9b5ff8..9ce4116 100644 --- a/assets/windows/alerts.yaml +++ b/assets/windows/alerts.yaml @@ -9,6 +9,7 @@ groups: expr: | 100 - (avg without (mode, core) (rate(windows_cpu_time_total{job=~".*windows.*", mode="idle"}[2m])) * 100) > 90 for: 15m + keep_firing_for: 5m labels: severity: warning - alert: WindowsMemoryHighUtilization @@ -21,6 +22,7 @@ groups: / windows_cs_physical_memory_bytes{job=~".*windows.*"}) * 100) > 90 for: 15m + keep_firing_for: 5m labels: severity: critical - alert: WindowsDiskAlmostOutOfSpace @@ -31,6 +33,7 @@ groups: expr: | 100 - ((windows_logical_disk_free_bytes{job=~".*windows.*"} ) / (windows_logical_disk_size_bytes{job=~".*windows.*"})) * 100 > 90 for: 15m + keep_firing_for: 5m labels: severity: critical - alert: WindowsServiceNotHealthy @@ -56,20 +59,22 @@ groups: - alert: WindowsNTPClientDelay annotations: description: | - 'Round-trip time of NTP client on instance {{ $labels.instance }} is greater than 1 second. Delay is {{ $value }} sec.' + Round-trip time of NTP client on instance {{ $labels.instance }} is greater than 1 second. Delay is {{ $value }} sec. summary: NTP client delay. expr: | windows_time_ntp_round_trip_delay_seconds{job=~".*windows.*"} > 1 for: 5m + keep_firing_for: 5m labels: severity: warning - alert: WindowsNTPTimeOffset annotations: description: | - 'NTP time offset for instance {{ $labels.instance }} is greater than 1 second. Offset is {{ $value }} sec.' + NTP time offset for instance {{ $labels.instance }} is greater than 1 second. Offset is {{ $value }} sec. summary: NTP time offset is too large. expr: | windows_time_computed_time_offset_seconds{job=~".*windows.*"} > 1 for: 5m + keep_firing_for: 5m labels: severity: warning diff --git a/assets/windows/dashboards/disks b/assets/windows/dashboards/disks index fd3a743..db34197 100644 --- a/assets/windows/dashboards/disks +++ b/assets/windows/dashboards/disks @@ -118,7 +118,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -240,7 +240,7 @@ "y": 1 }, "id": 3, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -415,7 +415,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -496,7 +496,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -569,7 +569,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -641,7 +641,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { diff --git a/assets/windows/dashboards/fleet b/assets/windows/dashboards/fleet index 72fa52e..23eab8f 100644 --- a/assets/windows/dashboards/fleet +++ b/assets/windows/dashboards/fleet @@ -138,7 +138,7 @@ { "targetBlank": false, "title": "Drill down to ${__field.name} ${__value.text}", - "url": "d/windows-overview?var-instance=${__data.fields.instance}&${__url_time_range}" + "url": "d/windows-overview?var-instance=${__data.fields.instance}&${__url_time_range}&${datasource:queryparam}" } ] } @@ -160,7 +160,7 @@ { "targetBlank": false, "title": "Filter by ${__field.name}", - "url": "d/windows-fleet?var-${__field.name}=${__value.text}&${__url_time_range}" + "url": "d/windows-fleet?var-${__field.name}=${__value.text}&${__url_time_range}&${datasource:queryparam}" } ] } @@ -411,7 +411,7 @@ "show": true } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -529,7 +529,7 @@ "id": "filterFieldsByName", "options": { "include": { - "pattern": "job 1|instance|product|^hostname$|Value.+" + "pattern": "job 1$|instance|product|^hostname$|Value.+" } } }, @@ -641,7 +641,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -746,7 +746,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -851,7 +851,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -956,7 +956,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -1017,7 +1017,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { diff --git a/assets/windows/dashboards/logs b/assets/windows/dashboards/logs index 4210cf3..6c76991 100644 --- a/assets/windows/dashboards/logs +++ b/assets/windows/dashboards/logs @@ -197,17 +197,19 @@ }, "gridPos": { "h": 6, - "w": 24 + "w": 24, + "x": 0, + "y": 0 }, "id": 1, - "interval": "30s", + "maxDataPoints": 100, "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -237,7 +239,9 @@ }, "gridPos": { "h": 18, - "w": 24 + "w": 24, + "x": 0, + "y": 18 }, "id": 2, "options": { @@ -247,7 +251,7 @@ "showTime": false, "wrapLogMessage": false }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -262,7 +266,7 @@ } ], "refresh": "1m", - "schemaVersion": 36, + "schemaVersion": 39, "tags": [ "windows" ], diff --git a/assets/windows/dashboards/overview b/assets/windows/dashboards/overview index 69a671d..5a871d8 100644 --- a/assets/windows/dashboards/overview +++ b/assets/windows/dashboards/overview @@ -121,7 +121,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -165,7 +165,7 @@ "fields": "/^hostname$/" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -210,7 +210,7 @@ "fields": "/^version$/" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -255,7 +255,7 @@ "fields": "/^product$/" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -299,7 +299,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -343,7 +343,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -388,7 +388,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -432,7 +432,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -490,7 +490,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -545,7 +545,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -604,7 +604,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -684,7 +684,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -775,7 +775,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -913,7 +913,7 @@ "y": 20 }, "id": 18, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -1095,7 +1095,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -1170,7 +1170,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { diff --git a/assets/windows/dashboards/system b/assets/windows/dashboards/system index 85a4f65..5f40b7f 100644 --- a/assets/windows/dashboards/system +++ b/assets/windows/dashboards/system @@ -115,7 +115,7 @@ ] } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -170,7 +170,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -286,7 +286,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -334,7 +334,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -382,7 +382,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -447,7 +447,7 @@ "fields": "/^timezone$/" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -500,7 +500,7 @@ }, "id": 9, "maxDataPoints": 50, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { @@ -550,7 +550,7 @@ "sort": "desc" } }, - "pluginVersion": "v10.0.0", + "pluginVersion": "v11.0.0", "targets": [ { "datasource": { diff --git a/site/content/apache-tomcat/_index.md b/site/content/apache-tomcat/_index.md index 0747a33..c2a43e7 100644 --- a/site/content/apache-tomcat/_index.md +++ b/site/content/apache-tomcat/_index.md @@ -27,7 +27,7 @@ annotations: last 5 minutes on {{$labels.instance}}, which is above the threshold of 80 percent. summary: The instance has a CPU usage higher than the configured threshold. expr: | - sum by (job, instance) (jvm_process_cpu_load{job=~"integrations/tomcat"}) > 80 + sum by (job,instance) (jvm_process_cpu_load{job=~"integrations/tomcat"}) > 80 for: 5m labels: severity: critical @@ -43,7 +43,7 @@ annotations: percent. summary: The instance has a higher memory usage than the configured threshold. expr: | - sum(jvm_memory_usage_used_bytes{job=~"integrations/tomcat"}) by (job, instance) / sum(jvm_physical_memory_bytes{job=~"integrations/tomcat"}) by (job, instance) * 100 > 80 + sum(jvm_memory_usage_used_bytes{job=~"integrations/tomcat"}) by (job,instance) / sum(jvm_physical_memory_bytes{job=~"integrations/tomcat"}) by (job,instance) * 100 > 80 for: 5m labels: severity: critical @@ -59,7 +59,7 @@ annotations: threshold of 5 percent. summary: There are a high number of request errors. expr: | - sum by (job, instance) (increase(tomcat_errorcount_total[5m]) / increase(tomcat_requestcount_total[5m]) * 100) > 5 + sum by (job,instance) (increase(tomcat_errorcount_total{job=~"integrations/tomcat"}[5m]) / increase(tomcat_requestcount_total{job=~"integrations/tomcat"}[5m]) * 100) > 5 for: 5m labels: severity: critical @@ -74,7 +74,7 @@ annotations: last 5 minutes on {{$labels.instance}}, which is above the threshold of 300ms. summary: The processing time has been moderately high. expr: | - sum by (job, instance) (increase(tomcat_processingtime_total[5m]) / increase(tomcat_requestcount_total[5m])) > 300 + sum by (job,instance) (increase(tomcat_processingtime_total{job=~"integrations/tomcat"}[5m]) / increase(tomcat_requestcount_total{job=~"integrations/tomcat"}[5m])) > 300 for: 5m labels: severity: warning diff --git a/site/content/cert-manager/_index.md b/site/content/cert-manager/_index.md index ccb8586..cff16e3 100644 --- a/site/content/cert-manager/_index.md +++ b/site/content/cert-manager/_index.md @@ -99,4 +99,4 @@ labels: Following dashboards are generated from mixins and hosted on github: -- [cert-manager](https://github.com/monitoring-mixins/website/blob/master/assets/cert-manager/dashboards/cert-manager.json) +- [overview](https://github.com/monitoring-mixins/website/blob/master/assets/cert-manager/dashboards/overview.json) diff --git a/site/content/cilium-enterprise/_index.md b/site/content/cilium-enterprise/_index.md index 2fdce9f..452f24c 100644 --- a/site/content/cilium-enterprise/_index.md +++ b/site/content/cilium-enterprise/_index.md @@ -56,6 +56,14 @@ Complete list of pregenerated alerts is available [here](https://github.com/moni ### Nodes +### Cilium + +### Clustermesh + +### Cilium + +### Kvstoremesh + ## Dashboards Following dashboards are generated from mixins and hosted on github: diff --git a/site/content/consul/_index.md b/site/content/consul/_index.md index 1b6be1b..698278f 100644 --- a/site/content/consul/_index.md +++ b/site/content/consul/_index.md @@ -60,8 +60,3 @@ labels: severity: critical {{< /code >}} -## Dashboards -Following dashboards are generated from mixins and hosted on github: - - -- [consul-overview](https://github.com/monitoring-mixins/website/blob/master/assets/consul/dashboards/consul-overview.json) diff --git a/site/content/cortex/_index.md b/site/content/cortex/_index.md index 843b9d5..bf69a7d 100644 --- a/site/content/cortex/_index.md +++ b/site/content/cortex/_index.md @@ -65,23 +65,6 @@ labels: severity: warning {{< /code >}} -##### CortexTableSyncFailure - -{{< code lang="yaml" >}} -alert: CortexTableSyncFailure -annotations: - message: | - {{ $labels.job }} is experiencing {{ printf "%.2f" $value }}% errors syncing tables. -expr: | - 100 * rate(cortex_table_manager_sync_duration_seconds_count{status_code!~"2.."}[15m]) - / - rate(cortex_table_manager_sync_duration_seconds_count[15m]) - > 10 -for: 30m -labels: - severity: critical -{{< /code >}} - ##### CortexQueriesIncorrect {{< code lang="yaml" >}} @@ -185,36 +168,6 @@ labels: severity: warning {{< /code >}} -##### CortexTransferFailed - -{{< code lang="yaml" >}} -alert: CortexTransferFailed -annotations: - message: | - {{ $labels.job }}/{{ $labels.instance }} transfer failed. -expr: | - max_over_time(cortex_shutdown_duration_seconds_count{op="transfer",status!="success"}[15m]) -for: 5m -labels: - severity: critical -{{< /code >}} - -##### CortexOldChunkInMemory - -{{< code lang="yaml" >}} -alert: CortexOldChunkInMemory -annotations: - message: | - {{ $labels.job }}/{{ $labels.instance }} has very old unflushed chunk in memory. -expr: | - (time() - cortex_oldest_unflushed_chunk_timestamp_seconds > 36000) - and - (cortex_oldest_unflushed_chunk_timestamp_seconds > 0) -for: 5m -labels: - severity: warning -{{< /code >}} - ##### CortexKVStoreFailure {{< code lang="yaml" >}} @@ -341,73 +294,6 @@ labels: severity: critical {{< /code >}} -### cortex_wal_alerts - -##### CortexWALCorruption - -{{< code lang="yaml" >}} -alert: CortexWALCorruption -annotations: - message: | - {{ $labels.job }}/{{ $labels.instance }} has a corrupted WAL or checkpoint. -expr: | - increase(cortex_ingester_wal_corruptions_total[5m]) > 0 -labels: - severity: critical -{{< /code >}} - -##### CortexCheckpointCreationFailed - -{{< code lang="yaml" >}} -alert: CortexCheckpointCreationFailed -annotations: - message: | - {{ $labels.job }}/{{ $labels.instance }} failed to create checkpoint. -expr: | - increase(cortex_ingester_checkpoint_creations_failed_total[10m]) > 0 -labels: - severity: warning -{{< /code >}} - -##### CortexCheckpointCreationFailed - -{{< code lang="yaml" >}} -alert: CortexCheckpointCreationFailed -annotations: - message: | - {{ $labels.job }}/{{ $labels.instance }} is failing to create checkpoint. -expr: | - increase(cortex_ingester_checkpoint_creations_failed_total[1h]) > 1 -labels: - severity: critical -{{< /code >}} - -##### CortexCheckpointDeletionFailed - -{{< code lang="yaml" >}} -alert: CortexCheckpointDeletionFailed -annotations: - message: | - {{ $labels.job }}/{{ $labels.instance }} failed to delete checkpoint. -expr: | - increase(cortex_ingester_checkpoint_deletions_failed_total[10m]) > 0 -labels: - severity: warning -{{< /code >}} - -##### CortexCheckpointDeletionFailed - -{{< code lang="yaml" >}} -alert: CortexCheckpointDeletionFailed -annotations: - message: | - {{ $labels.instance }} is failing to delete checkpoint. -expr: | - increase(cortex_ingester_checkpoint_deletions_failed_total[2h]) > 1 -labels: - severity: critical -{{< /code >}} - ### cortex-rollout-alerts ##### CortexRolloutStuck @@ -466,28 +352,6 @@ labels: ### cortex-provisioning -##### CortexProvisioningMemcachedTooSmall - -{{< code lang="yaml" >}} -alert: CortexProvisioningMemcachedTooSmall -annotations: - message: | - Chunk memcached cluster in {{ $labels.cluster }}/{{ $labels.namespace }} is too small, should be at least {{ printf "%.2f" $value }}GB. -expr: | - ( - 4 * - sum by (cluster, namespace) (cortex_ingester_memory_series * cortex_ingester_chunk_size_bytes_sum / cortex_ingester_chunk_size_bytes_count) - / 1e9 - ) - > - ( - sum by (cluster, namespace) (memcached_limit_bytes{job=~".+/memcached"}) / 1e9 - ) -for: 15m -labels: - severity: warning -{{< /code >}} - ##### CortexProvisioningTooManyActiveSeries {{< code lang="yaml" >}} @@ -496,7 +360,7 @@ annotations: message: | The number of in-memory series per ingester in {{ $labels.cluster }}/{{ $labels.namespace }} is too high. expr: | - avg by (cluster, namespace) (cortex_ingester_memory_series) > 1.6e6 + avg by (cluster, namespace) (cortex_ingester_memory_series) > 3.2e6 for: 2h labels: severity: warning @@ -510,7 +374,7 @@ annotations: message: | Ingesters in {{ $labels.cluster }}/{{ $labels.namespace }} ingest too many samples per second. expr: | - avg by (cluster, namespace) (rate(cortex_ingester_ingested_samples_total[1m])) > 80e3 + avg by (cluster, namespace) (rate(cortex_ingester_ingested_samples_total[1m])) > 160e3 for: 15m labels: severity: warning @@ -2329,7 +2193,7 @@ expr: | cortex_ingester_memory_series )[24h:] ) - / 1500000 + / 3000000 ) labels: deployment: ingester @@ -2343,7 +2207,7 @@ record: cluster_namespace_deployment_reason:required_replicas:count expr: | ceil( sum by (cluster, namespace) (cortex_overrides{limit_name="max_global_series_per_user"}) - * 3 * 0.59999999999999998 / 1500000 + * 3 * 0.59999999999999998 / 3000000 ) labels: deployment: ingester diff --git a/site/content/docker/_index.md b/site/content/docker/_index.md index f98382a..ae39e8a 100644 --- a/site/content/docker/_index.md +++ b/site/content/docker/_index.md @@ -10,9 +10,3 @@ A set of Grafana dashboards for Docker (based on cadvisor). Jsonnet source code is available at [github.com/grafana/jsonnet-libs](https://github.com/grafana/jsonnet-libs/tree/master/docker-mixin) {{< /panel >}} -## Dashboards -Following dashboards are generated from mixins and hosted on github: - - -- [docker-logs](https://github.com/monitoring-mixins/website/blob/master/assets/docker/dashboards/docker-logs.json) -- [docker](https://github.com/monitoring-mixins/website/blob/master/assets/docker/dashboards/docker.json) diff --git a/site/content/etcd/_index.md b/site/content/etcd/_index.md index 5eae88c..7444e42 100644 --- a/site/content/etcd/_index.md +++ b/site/content/etcd/_index.md @@ -227,7 +227,7 @@ annotations: as the writes to etcd will be disabled when it is full.' summary: etcd cluster database is running full. expr: | - (last_over_time(etcd_mvcc_db_total_size_in_bytes[5m]) / last_over_time(etcd_server_quota_backend_bytes[5m]))*100 > 95 + (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95 for: 10m labels: severity: critical @@ -243,7 +243,7 @@ annotations: on etcd instance {{ $labels.instance }}, please check as it might be disruptive.' summary: etcd cluster database growing very fast. expr: | - predict_linear(etcd_mvcc_db_total_size_in_bytes[4h], 4*60*60) > etcd_server_quota_backend_bytes + predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"} for: 10m labels: severity: warning @@ -261,7 +261,7 @@ annotations: runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation summary: etcd database size in use is less than 50% of the actual allocated storage. expr: | - (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes > 104857600 + (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600 for: 10m labels: severity: warning diff --git a/site/content/gluster/_index.md b/site/content/gluster/_index.md index 8f3c1c6..5101583 100644 --- a/site/content/gluster/_index.md +++ b/site/content/gluster/_index.md @@ -203,8 +203,3 @@ expr: | record: gluster:volume_capacity_total_bytes:sum {{< /code >}} -## Dashboards -Following dashboards are generated from mixins and hosted on github: - - -- [k8s-storage-resources-glusterfs-pv](https://github.com/monitoring-mixins/website/blob/master/assets/gluster/dashboards/k8s-storage-resources-glusterfs-pv.json) diff --git a/site/content/grafana/_index.md b/site/content/grafana/_index.md index cd9c7f0..de11eeb 100644 --- a/site/content/grafana/_index.md +++ b/site/content/grafana/_index.md @@ -26,8 +26,8 @@ annotations: message: '{{ $labels.namespace }}/{{ $labels.job }}/{{ $labels.handler }} is experiencing {{ $value | humanize }}% errors' expr: | - 100 * namespace_job_handler_statuscode:grafana_http_request_duration_seconds_count:rate5m{handler!~"/api/datasources/proxy/:id.*|/api/ds/query|/api/tsdb/query", status_code=~"5.."} - / ignoring (status_code) + 100 * sum without (status_code) (namespace_job_handler_statuscode:grafana_http_request_duration_seconds_count:rate5m{handler!~"/api/datasources/proxy/:id.*|/api/ds/query|/api/tsdb/query", status_code=~"5.."}) + / sum without (status_code) (namespace_job_handler_statuscode:grafana_http_request_duration_seconds_count:rate5m{handler!~"/api/datasources/proxy/:id.*|/api/ds/query|/api/tsdb/query"}) > 50 for: 5m diff --git a/site/content/haproxy/_index.md b/site/content/haproxy/_index.md index 44d6e08..2a5ac84 100644 --- a/site/content/haproxy/_index.md +++ b/site/content/haproxy/_index.md @@ -26,7 +26,7 @@ annotations: description: HAProxy {{$labels.job}} on {{$labels.instance}} is dropping logs. summary: HAProxy is dropping logs. expr: rate(haproxy_process_dropped_logs_total[5m]) != 0 -for: 5s +for: 10m labels: severity: critical {{< /code >}} @@ -40,7 +40,7 @@ annotations: has flapping checks. summary: HAProxy backend checks are flapping. expr: rate(haproxy_backend_check_up_down_total[5m]) != 0 -for: 1m +for: 10m labels: severity: critical {{< /code >}} @@ -54,7 +54,7 @@ annotations: has flapping checks. summary: HAProxy server checks are flapping. expr: rate(haproxy_server_check_up_down_total[5m]) != 0 -for: 1m +for: 10m labels: severity: critical {{< /code >}} diff --git a/site/content/jaeger/_index.md b/site/content/jaeger/_index.md index 725b415..2786765 100644 --- a/site/content/jaeger/_index.md +++ b/site/content/jaeger/_index.md @@ -23,8 +23,9 @@ Complete list of pregenerated alerts is available [here](https://github.com/moni {{< code lang="yaml" >}} alert: JaegerAgentUDPPacketsBeingDropped annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }} UDP packets per second. + summary: Jaeger agent is dropping UDP packets. expr: rate(jaeger_agent_thrift_udp_server_packets_dropped_total[1m]) > 1 for: 15m labels: @@ -36,8 +37,9 @@ labels: {{< code lang="yaml" >}} alert: JaegerAgentHTTPServerErrs annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is experiencing {{ printf "%.2f" $value }}% HTTP errors. + summary: Jaeger agent is experiencing HTTP errors. expr: 100 * sum(rate(jaeger_agent_http_server_errors_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_agent_http_server_total[1m])) by (instance, job, namespace)> 1 @@ -51,8 +53,9 @@ labels: {{< code lang="yaml" >}} alert: JaegerClientSpansDropped annotations: - message: | + description: | service {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }}% spans. + summary: Jaeger client is dropping spans. expr: 100 * sum(rate(jaeger_reporter_spans{result=~"dropped|err"}[1m])) by (instance, job, namespace) / sum(rate(jaeger_reporter_spans[1m])) by (instance, job, namespace)> 1 @@ -66,8 +69,9 @@ labels: {{< code lang="yaml" >}} alert: JaegerAgentSpansDropped annotations: - message: | + description: | agent {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }}% spans. + summary: Jaeger agent is dropping spans. expr: 100 * sum(rate(jaeger_agent_reporter_batches_failures_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_agent_reporter_batches_submitted_total[1m])) by (instance, job, namespace)> 1 @@ -81,8 +85,9 @@ labels: {{< code lang="yaml" >}} alert: JaegerCollectorQueueNotDraining annotations: - message: | + description: | collector {{ $labels.job }} {{ $labels.instance }} is not able to drain the queue. + summary: Jaeger collector is not able to drain the queue. expr: avg_over_time(jaeger_collector_queue_length[10m]) > 1000 for: 15m labels: @@ -94,8 +99,9 @@ labels: {{< code lang="yaml" >}} alert: JaegerCollectorDroppingSpans annotations: - message: | + description: | collector {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }}% spans. + summary: Jaeger collector is dropping spans. expr: 100 * sum(rate(jaeger_collector_spans_dropped_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_collector_spans_received_total[1m])) by (instance, job, namespace)> 1 @@ -109,8 +115,9 @@ labels: {{< code lang="yaml" >}} alert: JaegerSamplingUpdateFailing annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is failing {{ printf "%.2f" $value }}% in updating sampling policies. + summary: Jaeger's sampling update is failing. expr: 100 * sum(rate(jaeger_sampler_queries{result="err"}[1m])) by (instance, job, namespace) / sum(rate(jaeger_sampler_queries[1m])) by (instance, job, namespace)> 1 @@ -124,8 +131,9 @@ labels: {{< code lang="yaml" >}} alert: JaegerCollectorPersistenceSlow annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is slow at persisting spans. + summary: Jaeger collector is slow at persisting spans. expr: histogram_quantile(0.99, sum by (le) (rate(jaeger_collector_save_latency_bucket[1m]))) > 0.5 for: 15m @@ -138,8 +146,9 @@ labels: {{< code lang="yaml" >}} alert: JaegerThrottlingUpdateFailing annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is failing {{ printf "%.2f" $value }}% in updating throttling policies. + summary: Jaeger's throttling update is failing. expr: 100 * sum(rate(jaeger_throttler_updates{result="err"}[1m])) by (instance, job, namespace) / sum(rate(jaeger_throttler_updates[1m])) by (instance, job, namespace)> 1 @@ -153,8 +162,9 @@ labels: {{< code lang="yaml" >}} alert: JaegerQueryReqsFailing annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is seeing {{ printf "%.2f" $value }}% query errors on {{ $labels.operation }}. + summary: Jaeger queries are failing. expr: 100 * sum(rate(jaeger_query_requests_total{result="err"}[1m])) by (instance, job, namespace) / sum(rate(jaeger_query_requests_total[1m])) by (instance, job, namespace)> 1 @@ -168,8 +178,9 @@ labels: {{< code lang="yaml" >}} alert: JaegerCassandraWritesFailing annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is seeing {{ printf "%.2f" $value }}% query errors on {{ $labels.operation }}. + summary: Jaeger writes to Cassandra are failing. expr: 100 * sum(rate(jaeger_cassandra_errors_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_cassandra_attempts_total[1m])) by (instance, job, namespace)> 1 @@ -183,8 +194,9 @@ labels: {{< code lang="yaml" >}} alert: JaegerCassandraReadsFailing annotations: - message: | + description: | {{ $labels.job }} {{ $labels.instance }} is seeing {{ printf "%.2f" $value }}% query errors on {{ $labels.operation }}. + summary: Jaeger reads from Cassandra are failing. expr: 100 * sum(rate(jaeger_cassandra_read_errors_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_cassandra_read_attempts_total[1m])) by (instance, job, namespace)> 1 diff --git a/site/content/jvm/_index.md b/site/content/jvm/_index.md index c135fe0..a3f6ef4 100644 --- a/site/content/jvm/_index.md +++ b/site/content/jvm/_index.md @@ -10,31 +10,3 @@ title: jvm Jsonnet source code is available at [github.com/grafana/jsonnet-libs](https://github.com/grafana/jsonnet-libs/tree/master/jvm-mixin) {{< /panel >}} -## Alerts - -{{< panel style="warning" >}} -Complete list of pregenerated alerts is available [here](https://github.com/monitoring-mixins/website/blob/master/assets/jvm/alerts.yaml). -{{< /panel >}} - -### jvm - -##### JvmMemoryFillingUp - -{{< code lang="yaml" >}} -alert: JvmMemoryFillingUp -annotations: - description: JVM memory usage is at {{ printf "%%.0f" $value }} percent over the - last 5 minutes on {{$labels.instance}}, which is above the threshold of 80%. - summary: JVM memory filling up. -expr: | - jvm_memory_bytes_used / jvm_memory_bytes_max{area="heap"} > 0.8 -for: 5m -labels: - severity: warning -{{< /code >}} - -## Dashboards -Following dashboards are generated from mixins and hosted on github: - - -- [jvm-dashboard](https://github.com/monitoring-mixins/website/blob/master/assets/jvm/dashboards/jvm-dashboard.json) diff --git a/site/content/kafka/_index.md b/site/content/kafka/_index.md index f1ba6c5..3772933 100644 --- a/site/content/kafka/_index.md +++ b/site/content/kafka/_index.md @@ -10,160 +10,3 @@ title: kafka Jsonnet source code is available at [github.com/grafana/jsonnet-libs](https://github.com/grafana/jsonnet-libs/tree/master/kafka-mixin) {{< /panel >}} -## Alerts - -{{< panel style="warning" >}} -Complete list of pregenerated alerts is available [here](https://github.com/monitoring-mixins/website/blob/master/assets/kafka/alerts.yaml). -{{< /panel >}} - -### Kafka_Alerts - -##### KafkaOfflinePartitonCount - -{{< code lang="yaml" >}} -alert: KafkaOfflinePartitonCount -annotations: - description: Kafka cluster {{ $labels.kafka_cluster }} has {{ $value }} offline - partitions. After successful leader election, if the leader for partition dies, - then the partition moves to the OfflinePartition state. Offline partitions are - not available for reading and writing. Restart the brokers, if needed, and check - the logs for errors. - summary: Kafka has offline partitons. -expr: sum without(instance) (kafka_controller_kafkacontroller_offlinepartitionscount{job="integrations/kafka"}) - > 0 -for: 5m -labels: - severity: critical -{{< /code >}} - -##### KafkaUnderReplicatedPartitionCount - -{{< code lang="yaml" >}} -alert: KafkaUnderReplicatedPartitionCount -annotations: - description: Kafka instance {{ $labels.instance }} in cluster {{ $labels.kafka_cluster - }} has {{ $value }} under replicated partitons - summary: Kafka has under replicated partitons. -expr: | - sum without() (kafka_server_replicamanager_underreplicatedpartitions{job="integrations/kafka"}) > 0 -for: 5m -labels: - severity: critical -{{< /code >}} - -##### KafkaActiveController - -{{< code lang="yaml" >}} -alert: KafkaActiveController -annotations: - description: Kafka cluster {{ $labels.kafka_cluster }} has {{ $value }} broker(s) - reporting as the active controller in the last 5 minute interval. During steady - state there should be only one active controller per cluster. - summary: Kafka has no active controller. -expr: sum without(instance) (kafka_controller_kafkacontroller_activecontrollercount{job="integrations/kafka"}) - != 1 -for: 5m -labels: - severity: critical -{{< /code >}} - -##### KafkaUncleanLeaderElection - -{{< code lang="yaml" >}} -alert: KafkaUncleanLeaderElection -annotations: - description: Kafka cluster {{ $labels.kafka_cluster }} has {{ $value }} unclean - partition leader elections reported in the last 5 minute interval. When unclean - leader election is held among out-of-sync replicas, there is a possibility of - data loss if any messages were not synced prior to the loss of the former leader. - So if the number of unclean elections is greater than 0, investigate broker logs - to determine why leaders were re-elected, and look for WARN or ERROR messages. - Consider setting the broker configuration parameter unclean.leader.election.enable - to false so that a replica outside of the set of in-sync replicas is never elected - leader. - summary: Kafka has unclean leader elections. -expr: max without(instance) (rate(kafka_controller_controllerstats_uncleanleaderelectionspersec{job="integrations/kafka"}[5m])) - != 0 -for: 5m -labels: - severity: critical -{{< /code >}} - -##### KafkaISRExpandRate - -{{< code lang="yaml" >}} -alert: KafkaISRExpandRate -annotations: - description: Kafka instance {{ $labels.instance }} in cluster {{ $labels.kafka_cluster - }} ISR is expanding by {{ $value }} per second. If a broker goes down, ISR for - some of the partitions shrink. When that broker is up again, ISRs are expanded - once the replicas are fully caught up. Other than that, the expected value for - ISR expansion rate is 0. If ISR is expanding and shrinking frequently, adjust - Allowed replica lag. - summary: Kafka ISR Expansion Rate is expanding. -expr: | - sum without() (rate(kafka_server_replicamanager_isrexpandspersec{job="integrations/kafka"}[5m])) != 0 -for: 5m -labels: - severity: warning -{{< /code >}} - -##### KafkaISRShrinkRate - -{{< code lang="yaml" >}} -alert: KafkaISRShrinkRate -annotations: - description: Kafka instance {{ $labels.instance }} in cluster {{ $labels.kafka_cluster - }} ISR is shrinking by {{ $value }} per second. If a broker goes down, ISR for - some of the partitions shrink. When that broker is up again, ISRs are expanded - once the replicas are fully caught up. Other than that, the expected value for - ISR shrink rate is 0. If ISR is expanding and shrinking frequently, adjust Allowed - replica lag. - summary: Kafka ISR Expansion Rate is shrinking. -expr: | - sum without() (rate(kafka_server_replicamanager_isrshrinkspersec{job="integrations/kafka"}[5m])) != 0 -for: 5m -labels: - severity: warning -{{< /code >}} - -##### KafkaBrokerCount - -{{< code lang="yaml" >}} -alert: KafkaBrokerCount -annotations: - description: Kafka cluster {{ $labels.kafka_cluster }} broker count is 0. - summary: Kafka has no Brokers online. -expr: count without(instance) (kafka_server_kafkaserver_brokerstate{job="integrations/kafka"}) - == 0 -for: 5m -labels: - severity: critical -{{< /code >}} - -##### KafkaZookeeperSyncConnect - -{{< code lang="yaml" >}} -alert: KafkaZookeeperSyncConnect -annotations: - description: Kafka instance {{ $labels.instance }} in cluster {{ $labels.kafka_cluster - }} Zookeeper Sync Disconected. - summary: Kafka Zookeeper Sync Disconected. -expr: | - avg without() (kafka_server_sessionexpirelistener_zookeepersyncconnectspersec{job="integrations/kafka"}) < 0 -for: 5m -labels: - severity: warning -{{< /code >}} - -## Dashboards -Following dashboards are generated from mixins and hosted on github: - - -- [connect-overview](https://github.com/monitoring-mixins/website/blob/master/assets/kafka/dashboards/connect-overview.json) -- [kafka-ksqldb-overview](https://github.com/monitoring-mixins/website/blob/master/assets/kafka/dashboards/kafka-ksqldb-overview.json) -- [kafka-lag-overview](https://github.com/monitoring-mixins/website/blob/master/assets/kafka/dashboards/kafka-lag-overview.json) -- [kafka-overview](https://github.com/monitoring-mixins/website/blob/master/assets/kafka/dashboards/kafka-overview.json) -- [kafka-topics](https://github.com/monitoring-mixins/website/blob/master/assets/kafka/dashboards/kafka-topics.json) -- [schema-registry-overview](https://github.com/monitoring-mixins/website/blob/master/assets/kafka/dashboards/schema-registry-overview.json) -- [zookeeper-overview](https://github.com/monitoring-mixins/website/blob/master/assets/kafka/dashboards/zookeeper-overview.json) diff --git a/site/content/kubernetes/_index.md b/site/content/kubernetes/_index.md index 790f98a..fb6b1c6 100644 --- a/site/content/kubernetes/_index.md +++ b/site/content/kubernetes/_index.md @@ -178,7 +178,7 @@ annotations: summary: StatefulSet update has not been rolled out. expr: | ( - max without (revision) ( + max by(namespace, statefulset, job, cluster) ( kube_statefulset_status_current_revision{job="kube-state-metrics"} unless kube_statefulset_status_update_revision{job="kube-state-metrics"} @@ -525,9 +525,9 @@ annotations: runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh summary: Processes experience elevated CPU throttling. expr: | - sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace) + sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (cluster, container, pod, namespace) / - sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace) + sum(increase(container_cpu_cfs_periods_total{}[5m])) by (cluster, container, pod, namespace) > ( 25 / 100 ) for: 15m labels: @@ -543,8 +543,8 @@ https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md alert: KubePersistentVolumeFillingUp annotations: description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} - in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} - free. + in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ + . }} {{- end }} is only {{ $value | humanizePercentage }} free. runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup summary: PersistentVolume is filling up. expr: | @@ -555,9 +555,9 @@ expr: | ) < 0.03 and kubelet_volume_stats_used_bytes{job="kubelet"} > 0 - unless on(namespace, persistentvolumeclaim) + unless on(cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 - unless on(namespace, persistentvolumeclaim) + unless on(cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 for: 1m labels: @@ -571,8 +571,9 @@ https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md alert: KubePersistentVolumeFillingUp annotations: description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim - }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. - Currently {{ $value | humanizePercentage }} is available. + }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster + {{ . }} {{- end }} is expected to fill up within four days. Currently {{ $value + | humanizePercentage }} is available. runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup summary: PersistentVolume is filling up. expr: | @@ -585,9 +586,9 @@ expr: | kubelet_volume_stats_used_bytes{job="kubelet"} > 0 and predict_linear(kubelet_volume_stats_available_bytes{job="kubelet"}[6h], 4 * 24 * 3600) < 0 - unless on(namespace, persistentvolumeclaim) + unless on(cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 - unless on(namespace, persistentvolumeclaim) + unless on(cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 for: 1h labels: @@ -601,8 +602,8 @@ https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md alert: KubePersistentVolumeInodesFillingUp annotations: description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} - in Namespace {{ $labels.namespace }} only has {{ $value | humanizePercentage }} - free inodes. + in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ + . }} {{- end }} only has {{ $value | humanizePercentage }} free inodes. runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeinodesfillingup summary: PersistentVolumeInodes are filling up. expr: | @@ -613,9 +614,9 @@ expr: | ) < 0.03 and kubelet_volume_stats_inodes_used{job="kubelet"} > 0 - unless on(namespace, persistentvolumeclaim) + unless on(cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 - unless on(namespace, persistentvolumeclaim) + unless on(cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 for: 1m labels: @@ -629,8 +630,9 @@ https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md alert: KubePersistentVolumeInodesFillingUp annotations: description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim - }} in Namespace {{ $labels.namespace }} is expected to run out of inodes within - four days. Currently {{ $value | humanizePercentage }} of its inodes are free. + }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster + {{ . }} {{- end }} is expected to run out of inodes within four days. Currently + {{ $value | humanizePercentage }} of its inodes are free. runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeinodesfillingup summary: PersistentVolumeInodes are filling up. expr: | @@ -643,9 +645,9 @@ expr: | kubelet_volume_stats_inodes_used{job="kubelet"} > 0 and predict_linear(kubelet_volume_stats_inodes_free{job="kubelet"}[6h], 4 * 24 * 3600) < 0 - unless on(namespace, persistentvolumeclaim) + unless on(cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 - unless on(namespace, persistentvolumeclaim) + unless on(cluster, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 for: 1h labels: @@ -658,8 +660,8 @@ https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md {{< code lang="yaml" >}} alert: KubePersistentVolumeErrors annotations: - description: The persistent volume {{ $labels.persistentvolume }} has status {{ - $labels.phase }}. + description: The persistent volume {{ $labels.persistentvolume }} {{ with $labels.cluster + -}} on Cluster {{ . }} {{- end }} has status {{ $labels.phase }}. runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors summary: PersistentVolume is having issues with provisioning. expr: | @@ -720,9 +722,9 @@ annotations: runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn summary: The API server is burning too much error budget. expr: | - sum(apiserver_request:burnrate1h) > (14.40 * 0.01000) - and - sum(apiserver_request:burnrate5m) > (14.40 * 0.01000) + sum by(cluster) (apiserver_request:burnrate1h) > (14.40 * 0.01000) + and on(cluster) + sum by(cluster) (apiserver_request:burnrate5m) > (14.40 * 0.01000) for: 2m labels: long: 1h @@ -740,9 +742,9 @@ annotations: runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn summary: The API server is burning too much error budget. expr: | - sum(apiserver_request:burnrate6h) > (6.00 * 0.01000) - and - sum(apiserver_request:burnrate30m) > (6.00 * 0.01000) + sum by(cluster) (apiserver_request:burnrate6h) > (6.00 * 0.01000) + and on(cluster) + sum by(cluster) (apiserver_request:burnrate30m) > (6.00 * 0.01000) for: 15m labels: long: 6h @@ -760,9 +762,9 @@ annotations: runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn summary: The API server is burning too much error budget. expr: | - sum(apiserver_request:burnrate1d) > (3.00 * 0.01000) - and - sum(apiserver_request:burnrate2h) > (3.00 * 0.01000) + sum by(cluster) (apiserver_request:burnrate1d) > (3.00 * 0.01000) + and on(cluster) + sum by(cluster) (apiserver_request:burnrate2h) > (3.00 * 0.01000) for: 1h labels: long: 1d @@ -780,9 +782,9 @@ annotations: runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn summary: The API server is burning too much error budget. expr: | - sum(apiserver_request:burnrate3d) > (1.00 * 0.01000) - and - sum(apiserver_request:burnrate6h) > (1.00 * 0.01000) + sum by(cluster) (apiserver_request:burnrate3d) > (1.00 * 0.01000) + and on(cluster) + sum by(cluster) (apiserver_request:burnrate6h) > (1.00 * 0.01000) for: 3h labels: long: 3d @@ -803,7 +805,7 @@ annotations: runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration summary: Client certificate is about to expire. expr: | - apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 604800 + apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(cluster, job) histogram_quantile(0.01, sum by (cluster, job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 604800 for: 5m labels: severity: warning @@ -820,7 +822,7 @@ annotations: runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration summary: Client certificate is about to expire. expr: | - apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 86400 + apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(cluster, job) histogram_quantile(0.01, sum by (cluster, job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 86400 for: 5m labels: severity: critical @@ -888,7 +890,7 @@ annotations: summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests. expr: | - sum(rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="kube-apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m])) ) > 0.20 + sum by(cluster) (rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m])) / ( sum by(cluster) (rate(apiserver_request_total{job="kube-apiserver"}[10m])) + sum by(cluster) (rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m])) ) > 0.20 for: 5m labels: severity: warning @@ -1176,403 +1178,6 @@ labels: Complete list of pregenerated recording rules is available [here](https://github.com/monitoring-mixins/website/blob/master/assets/kubernetes/rules.yaml). {{< /panel >}} -### kube-apiserver-burnrate.rules - -##### apiserver_request:burnrate1d - -{{< code lang="yaml" >}} -expr: | - ( - ( - # too slow - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d])) - - - ( - ( - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d])) - or - vector(0) - ) - + - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d])) - + - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d])) - ) - ) - + - # errors - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) - ) - / - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[1d])) -labels: - verb: read -record: apiserver_request:burnrate1d -{{< /code >}} - -##### apiserver_request:burnrate1h - -{{< code lang="yaml" >}} -expr: | - ( - ( - # too slow - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h])) - - - ( - ( - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h])) - or - vector(0) - ) - + - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h])) - + - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h])) - ) - ) - + - # errors - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) - ) - / - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[1h])) -labels: - verb: read -record: apiserver_request:burnrate1h -{{< /code >}} - -##### apiserver_request:burnrate2h - -{{< code lang="yaml" >}} -expr: | - ( - ( - # too slow - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h])) - - - ( - ( - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h])) - or - vector(0) - ) - + - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h])) - + - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h])) - ) - ) - + - # errors - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) - ) - / - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[2h])) -labels: - verb: read -record: apiserver_request:burnrate2h -{{< /code >}} - -##### apiserver_request:burnrate30m - -{{< code lang="yaml" >}} -expr: | - ( - ( - # too slow - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m])) - - - ( - ( - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m])) - or - vector(0) - ) - + - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m])) - + - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m])) - ) - ) - + - # errors - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) - ) - / - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[30m])) -labels: - verb: read -record: apiserver_request:burnrate30m -{{< /code >}} - -##### apiserver_request:burnrate3d - -{{< code lang="yaml" >}} -expr: | - ( - ( - # too slow - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d])) - - - ( - ( - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d])) - or - vector(0) - ) - + - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d])) - + - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d])) - ) - ) - + - # errors - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) - ) - / - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[3d])) -labels: - verb: read -record: apiserver_request:burnrate3d -{{< /code >}} - -##### apiserver_request:burnrate5m - -{{< code lang="yaml" >}} -expr: | - ( - ( - # too slow - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m])) - - - ( - ( - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m])) - or - vector(0) - ) - + - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m])) - + - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m])) - ) - ) - + - # errors - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) - ) - / - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[5m])) -labels: - verb: read -record: apiserver_request:burnrate5m -{{< /code >}} - -##### apiserver_request:burnrate6h - -{{< code lang="yaml" >}} -expr: | - ( - ( - # too slow - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h])) - - - ( - ( - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h])) - or - vector(0) - ) - + - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h])) - + - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h])) - ) - ) - + - # errors - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) - ) - / - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[6h])) -labels: - verb: read -record: apiserver_request:burnrate6h -{{< /code >}} - -##### apiserver_request:burnrate1d - -{{< code lang="yaml" >}} -expr: | - ( - ( - # too slow - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d])) - - - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d])) - ) - + - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) - ) - / - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) -labels: - verb: write -record: apiserver_request:burnrate1d -{{< /code >}} - -##### apiserver_request:burnrate1h - -{{< code lang="yaml" >}} -expr: | - ( - ( - # too slow - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h])) - - - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h])) - ) - + - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) - ) - / - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) -labels: - verb: write -record: apiserver_request:burnrate1h -{{< /code >}} - -##### apiserver_request:burnrate2h - -{{< code lang="yaml" >}} -expr: | - ( - ( - # too slow - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h])) - - - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h])) - ) - + - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) - ) - / - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) -labels: - verb: write -record: apiserver_request:burnrate2h -{{< /code >}} - -##### apiserver_request:burnrate30m - -{{< code lang="yaml" >}} -expr: | - ( - ( - # too slow - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m])) - - - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m])) - ) - + - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) - ) - / - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) -labels: - verb: write -record: apiserver_request:burnrate30m -{{< /code >}} - -##### apiserver_request:burnrate3d - -{{< code lang="yaml" >}} -expr: | - ( - ( - # too slow - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d])) - - - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d])) - ) - + - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) - ) - / - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) -labels: - verb: write -record: apiserver_request:burnrate3d -{{< /code >}} - -##### apiserver_request:burnrate5m - -{{< code lang="yaml" >}} -expr: | - ( - ( - # too slow - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m])) - - - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m])) - ) - + - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) - ) - / - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) -labels: - verb: write -record: apiserver_request:burnrate5m -{{< /code >}} - -##### apiserver_request:burnrate6h - -{{< code lang="yaml" >}} -expr: | - ( - ( - # too slow - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h])) - - - sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h])) - ) - + - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) - ) - / - sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) -labels: - verb: write -record: apiserver_request:burnrate6h -{{< /code >}} - -### kube-apiserver-histogram.rules - -##### cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile - -{{< code lang="yaml" >}} -expr: | - histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 -labels: - quantile: "0.99" - verb: read -record: cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile -{{< /code >}} - -##### cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile - -{{< code lang="yaml" >}} -expr: | - histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_slo_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 -labels: - quantile: "0.99" - verb: write -record: cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile -{{< /code >}} - ### kube-apiserver-availability.rules ##### code_verb:apiserver_request_total:increase30d @@ -1603,36 +1208,36 @@ labels: record: code:apiserver_request_total:increase30d {{< /code >}} -##### cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase1h +##### cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h {{< code lang="yaml" >}} expr: | - sum by (cluster, verb, scope) (increase(apiserver_request_slo_duration_seconds_count{job="kube-apiserver"}[1h])) -record: cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase1h + sum by (cluster, verb, scope, le) (increase(apiserver_request_sli_duration_seconds_bucket[1h])) +record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h {{< /code >}} -##### cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d +##### cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d {{< code lang="yaml" >}} expr: | - sum by (cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase1h[30d]) * 24 * 30) -record: cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d + sum by (cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h[30d]) * 24 * 30) +record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d {{< /code >}} -##### cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase1h +##### cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h {{< code lang="yaml" >}} expr: | - sum by (cluster, verb, scope, le) (increase(apiserver_request_slo_duration_seconds_bucket[1h])) -record: cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase1h + sum by (cluster, verb, scope) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h{le="+Inf"}) +record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h {{< /code >}} -##### cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d +##### cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d {{< code lang="yaml" >}} expr: | - sum by (cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase1h[30d]) * 24 * 30) -record: cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d + sum by (cluster, verb, scope) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{le="+Inf"} * 24 * 30) +record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d {{< /code >}} ##### apiserver_request:availability30d @@ -1642,24 +1247,24 @@ expr: | 1 - ( ( # write too slow - sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) - - sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) ) + ( # read too slow - sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"LIST|GET"}) - ( ( - sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) or vector(0) ) + - sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + - sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) ) ) + # errors @@ -1677,19 +1282,19 @@ record: apiserver_request:availability30d {{< code lang="yaml" >}} expr: | 1 - ( - sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"LIST|GET"}) - ( # too slow ( - sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) or vector(0) ) + - sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + - sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) ) + # errors @@ -1709,9 +1314,9 @@ expr: | 1 - ( ( # too slow - sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) - - sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) ) + # errors @@ -1776,7 +1381,404 @@ expr: | record: code_verb:apiserver_request_total:increase1h {{< /code >}} -### k8s.rules +### kube-apiserver-burnrate.rules + +##### apiserver_request:burnrate1d + +{{< code lang="yaml" >}} +expr: | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d])) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[1d])) +labels: + verb: read +record: apiserver_request:burnrate1d +{{< /code >}} + +##### apiserver_request:burnrate1h + +{{< code lang="yaml" >}} +expr: | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h])) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[1h])) +labels: + verb: read +record: apiserver_request:burnrate1h +{{< /code >}} + +##### apiserver_request:burnrate2h + +{{< code lang="yaml" >}} +expr: | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h])) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[2h])) +labels: + verb: read +record: apiserver_request:burnrate2h +{{< /code >}} + +##### apiserver_request:burnrate30m + +{{< code lang="yaml" >}} +expr: | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m])) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[30m])) +labels: + verb: read +record: apiserver_request:burnrate30m +{{< /code >}} + +##### apiserver_request:burnrate3d + +{{< code lang="yaml" >}} +expr: | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d])) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[3d])) +labels: + verb: read +record: apiserver_request:burnrate3d +{{< /code >}} + +##### apiserver_request:burnrate5m + +{{< code lang="yaml" >}} +expr: | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m])) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[5m])) +labels: + verb: read +record: apiserver_request:burnrate5m +{{< /code >}} + +##### apiserver_request:burnrate6h + +{{< code lang="yaml" >}} +expr: | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h])) + + + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[6h])) +labels: + verb: read +record: apiserver_request:burnrate6h +{{< /code >}} + +##### apiserver_request:burnrate1d + +{{< code lang="yaml" >}} +expr: | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) +labels: + verb: write +record: apiserver_request:burnrate1d +{{< /code >}} + +##### apiserver_request:burnrate1h + +{{< code lang="yaml" >}} +expr: | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) +labels: + verb: write +record: apiserver_request:burnrate1h +{{< /code >}} + +##### apiserver_request:burnrate2h + +{{< code lang="yaml" >}} +expr: | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) +labels: + verb: write +record: apiserver_request:burnrate2h +{{< /code >}} + +##### apiserver_request:burnrate30m + +{{< code lang="yaml" >}} +expr: | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) +labels: + verb: write +record: apiserver_request:burnrate30m +{{< /code >}} + +##### apiserver_request:burnrate3d + +{{< code lang="yaml" >}} +expr: | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) +labels: + verb: write +record: apiserver_request:burnrate3d +{{< /code >}} + +##### apiserver_request:burnrate5m + +{{< code lang="yaml" >}} +expr: | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) +labels: + verb: write +record: apiserver_request:burnrate5m +{{< /code >}} + +##### apiserver_request:burnrate6h + +{{< code lang="yaml" >}} +expr: | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) +labels: + verb: write +record: apiserver_request:burnrate6h +{{< /code >}} + +### kube-apiserver-histogram.rules + +##### cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile + +{{< code lang="yaml" >}} +expr: | + histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 +labels: + quantile: "0.99" + verb: read +record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile +{{< /code >}} + +##### cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile + +{{< code lang="yaml" >}} +expr: | + histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 +labels: + quantile: "0.99" + verb: write +record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile +{{< /code >}} + +### k8s.rules.container_cpu_usage_seconds_total ##### node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate @@ -1790,6 +1792,8 @@ expr: | record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate {{< /code >}} +### k8s.rules.container_memory_working_set_bytes + ##### node_namespace_pod_container:container_memory_working_set_bytes {{< code lang="yaml" >}} @@ -1801,6 +1805,8 @@ expr: | record: node_namespace_pod_container:container_memory_working_set_bytes {{< /code >}} +### k8s.rules.container_memory_rss + ##### node_namespace_pod_container:container_memory_rss {{< code lang="yaml" >}} @@ -1812,6 +1818,8 @@ expr: | record: node_namespace_pod_container:container_memory_rss {{< /code >}} +### k8s.rules.container_memory_cache + ##### node_namespace_pod_container:container_memory_cache {{< code lang="yaml" >}} @@ -1823,6 +1831,8 @@ expr: | record: node_namespace_pod_container:container_memory_cache {{< /code >}} +### k8s.rules.container_memory_swap + ##### node_namespace_pod_container:container_memory_swap {{< code lang="yaml" >}} @@ -1834,6 +1844,8 @@ expr: | record: node_namespace_pod_container:container_memory_swap {{< /code >}} +### k8s.rules.container_memory_requests + ##### cluster:namespace:pod_memory:active:kube_pod_container_resource_requests {{< code lang="yaml" >}} @@ -1861,6 +1873,8 @@ expr: | record: namespace_memory:kube_pod_container_resource_requests:sum {{< /code >}} +### k8s.rules.container_cpu_requests + ##### cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests {{< code lang="yaml" >}} @@ -1888,6 +1902,8 @@ expr: | record: namespace_cpu:kube_pod_container_resource_requests:sum {{< /code >}} +### k8s.rules.container_memory_limits + ##### cluster:namespace:pod_memory:active:kube_pod_container_resource_limits {{< code lang="yaml" >}} @@ -1915,6 +1931,8 @@ expr: | record: namespace_memory:kube_pod_container_resource_limits:sum {{< /code >}} +### k8s.rules.container_cpu_limits + ##### cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits {{< code lang="yaml" >}} @@ -1942,6 +1960,8 @@ expr: | record: namespace_cpu:kube_pod_container_resource_limits:sum {{< /code >}} +### k8s.rules.pod_owner + ##### namespace_workload_pod:kube_pod_owner:relabel {{< code lang="yaml" >}} @@ -2120,8 +2140,8 @@ record: 'node_namespace_pod:kube_pod_info:' expr: | count by (cluster, node) ( node_cpu_seconds_total{mode="idle",job="node-exporter"} - * on (namespace, pod) group_left(node) - topk by(namespace, pod) (1, node_namespace_pod:kube_pod_info:) + * on (cluster, namespace, pod) group_left(node) + topk by(cluster, namespace, pod) (1, node_namespace_pod:kube_pod_info:) ) record: node:node_num_cpu:sum {{< /code >}} diff --git a/site/content/loki/_index.md b/site/content/loki/_index.md index 8bdb39a..e4f414f 100644 --- a/site/content/loki/_index.md +++ b/site/content/loki/_index.md @@ -23,12 +23,13 @@ Complete list of pregenerated alerts is available [here](https://github.com/moni {{< code lang="yaml" >}} alert: LokiRequestErrors annotations: - message: | - {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors. + description: | + {{ $labels.cluster }} {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors. + summary: Loki request error rate is high. expr: | - 100 * sum(rate(loki_request_duration_seconds_count{status_code=~"5.."}[2m])) by (namespace, job, route) + 100 * sum(rate(loki_request_duration_seconds_count{status_code=~"5.."}[2m])) by (cluster, namespace, job, route) / - sum(rate(loki_request_duration_seconds_count[2m])) by (namespace, job, route) + sum(rate(loki_request_duration_seconds_count[2m])) by (cluster, namespace, job, route) > 10 for: 15m labels: @@ -40,10 +41,11 @@ labels: {{< code lang="yaml" >}} alert: LokiRequestPanics annotations: - message: | - {{ $labels.job }} is experiencing {{ printf "%.2f" $value }}% increase of panics. + description: | + {{ $labels.cluster }} {{ $labels.job }} is experiencing {{ printf "%.2f" $value }}% increase of panics. + summary: Loki requests are causing code panics. expr: | - sum(increase(loki_panic_total[10m])) by (namespace, job) > 0 + sum(increase(loki_panic_total[10m])) by (cluster, namespace, job) > 0 labels: severity: critical {{< /code >}} @@ -53,8 +55,9 @@ labels: {{< code lang="yaml" >}} alert: LokiRequestLatency annotations: - message: | - {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency. + description: | + {{ $labels.cluster }} {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency. + summary: Loki request error latency is high. expr: | cluster_namespace_job_route:loki_request_duration_seconds:99quantile{route!~"(?i).*tail.*|/schedulerpb.SchedulerForQuerier/QuerierLoop"} > 1 for: 15m @@ -67,15 +70,59 @@ labels: {{< code lang="yaml" >}} alert: LokiTooManyCompactorsRunning annotations: - message: | + description: | {{ $labels.cluster }} {{ $labels.namespace }} has had {{ printf "%.0f" $value }} compactors running for more than 5m. Only one compactor should run at a time. + summary: Loki deployment is running more than one compactor. expr: | - sum(loki_boltdb_shipper_compactor_running) by (namespace, cluster) > 1 + sum(loki_boltdb_shipper_compactor_running) by (cluster, namespace) > 1 for: 5m labels: severity: warning {{< /code >}} +##### LokiCompactorHasNotSuccessfullyRunCompaction + +{{< code lang="yaml" >}} +alert: LokiCompactorHasNotSuccessfullyRunCompaction +annotations: + description: | + {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3 hours since the last compaction. This may indicate a problem with the compactor. + summary: Loki compaction has not run in the last 3 hours since the last compaction. +expr: | + # The "last successful run" metric is updated even if the compactor owns no tenants, + # so this alert correctly doesn't fire if compactor has nothing to do. + min ( + time() - (loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{} > 0) + ) + by (cluster, namespace) + > 60 * 60 * 3 +for: 1h +labels: + severity: critical +{{< /code >}} + +##### LokiCompactorHasNotSuccessfullyRunCompaction + +{{< code lang="yaml" >}} +alert: LokiCompactorHasNotSuccessfullyRunCompaction +annotations: + description: | + {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3h since startup. This may indicate a problem with the compactor. + summary: Loki compaction has not run in the last 3h since startup. +expr: | + # The "last successful run" metric is updated even if the compactor owns no tenants, + # so this alert correctly doesn't fire if compactor has nothing to do. + max( + max_over_time( + loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{}[3h] + ) + ) by (cluster, namespace) + == 0 +for: 1h +labels: + severity: critical +{{< /code >}} + ## Recording rules {{< panel style="warning" >}} @@ -227,6 +274,8 @@ record: cluster_namespace_job_route:loki_request_duration_seconds_count:sum_rate Following dashboards are generated from mixins and hosted on github: +- [loki-bloom-build](https://github.com/monitoring-mixins/website/blob/master/assets/loki/dashboards/loki-bloom-build.json) +- [loki-bloom-gateway](https://github.com/monitoring-mixins/website/blob/master/assets/loki/dashboards/loki-bloom-gateway.json) - [loki-chunks](https://github.com/monitoring-mixins/website/blob/master/assets/loki/dashboards/loki-chunks.json) - [loki-deletion](https://github.com/monitoring-mixins/website/blob/master/assets/loki/dashboards/loki-deletion.json) - [loki-logs](https://github.com/monitoring-mixins/website/blob/master/assets/loki/dashboards/loki-logs.json) diff --git a/site/content/memcached/_index.md b/site/content/memcached/_index.md index 5100773..8381f2f 100644 --- a/site/content/memcached/_index.md +++ b/site/content/memcached/_index.md @@ -63,6 +63,21 @@ labels: severity: critical {{< /code >}} +##### MemcachedOutOfMemoryErrors + +{{< code lang="yaml" >}} +alert: MemcachedOutOfMemoryErrors +annotations: + description: Memcached instance {{ $labels.job }} / {{ $labels.instance }} has OutOfMemory + errors for at least 15 minutes, current rate is {{ printf "%0.0f" $value }} + summary: Memcached has OutOfMemory errors. +expr: | + sum without (slab) (rate(memcached_slab_items_outofmemory_total[5m])) > 0 +for: 15m +labels: + severity: warning +{{< /code >}} + ## Dashboards Following dashboards are generated from mixins and hosted on github: diff --git a/site/content/mongodb/_index.md b/site/content/mongodb/_index.md index a0fbb2a..ce1b26d 100644 --- a/site/content/mongodb/_index.md +++ b/site/content/mongodb/_index.md @@ -29,7 +29,19 @@ annotations: LABELS = {{ $labels }} summary: MongoDB Instance is Down. expr: mongodb_up == 0 -for: 0m +for: 5m +labels: + severity: critical +{{< /code >}} + +##### MongodbReplicaMemberUnhealthy + +{{< code lang="yaml" >}} +alert: MongodbReplicaMemberUnhealthy +annotations: + description: Mongodb replica member unhealthy (instance {{ $labels.instance }}) + summary: MongoDB replica member unhealthy. +expr: mongodb_mongod_replset_member_health == 0 labels: severity: critical {{< /code >}} @@ -44,9 +56,8 @@ annotations: VALUE = {{ $value }} LABELS = {{ $labels }} summary: MongoDB replication lag is exceeding the threshold. -expr: mongodb_mongod_replset_member_optime_date{state="PRIMARY"} - ON (set) mongodb_mongod_replset_member_optime_date{state="SECONDARY"} - > 10 -for: 0m +expr: mongodb_mongod_replset_member_replication_lag{state="SECONDARY"} > 60 +for: 5m labels: severity: critical {{< /code >}} @@ -64,7 +75,7 @@ annotations: expr: (avg(mongodb_mongod_replset_oplog_tail_timestamp - mongodb_mongod_replset_oplog_head_timestamp) - (avg(mongodb_mongod_replset_member_optime_date{state="PRIMARY"}) - avg(mongodb_mongod_replset_member_optime_date{state="SECONDARY"}))) <= 0 -for: 0m +for: 5m labels: severity: critical {{< /code >}} @@ -135,6 +146,38 @@ labels: severity: warning {{< /code >}} +##### MongodbReadRequestsQueueingUp + +{{< code lang="yaml" >}} +alert: MongodbReadRequestsQueueingUp +annotations: + description: |- + MongoDB requests are queuing up + VALUE = {{ $value }} + LABELS = {{ $labels }} + summary: MongoDB read requests queuing up. +expr: delta(mongodb_mongod_global_lock_current_queue{type="reader"}[1m]) > 0 +for: 5m +labels: + severity: warning +{{< /code >}} + +##### MongodbWriteRequestsQueueingUp + +{{< code lang="yaml" >}} +alert: MongodbWriteRequestsQueueingUp +annotations: + description: |- + MongoDB write requests are queueing up + VALUE = {{ $value }} + LABELS = {{ $labels }} + summary: MongoDB write requests queuing up. +expr: delta(mongodb_mongod_global_lock_current_queue{type="writer"}[1m]) > 0 +for: 5m +labels: + severity: warning +{{< /code >}} + ## Dashboards Following dashboards are generated from mixins and hosted on github: diff --git a/site/content/nginx/_index.md b/site/content/nginx/_index.md index 523c104..cfb1ebf 100644 --- a/site/content/nginx/_index.md +++ b/site/content/nginx/_index.md @@ -14,4 +14,5 @@ Jsonnet source code is available at [github.com/grafana/jsonnet-libs](https://gi Following dashboards are generated from mixins and hosted on github: -- [nginx-overview](https://github.com/monitoring-mixins/website/blob/master/assets/nginx/dashboards/nginx-overview.json) +- [nginx-logs](https://github.com/monitoring-mixins/website/blob/master/assets/nginx/dashboards/nginx-logs.json) +- [nginx-metrics](https://github.com/monitoring-mixins/website/blob/master/assets/nginx/dashboards/nginx-metrics.json) diff --git a/site/content/node-exporter/_index.md b/site/content/node-exporter/_index.md index 3463245..d91d8fa 100644 --- a/site/content/node-exporter/_index.md +++ b/site/content/node-exporter/_index.md @@ -418,7 +418,7 @@ labels: alert: NodeDiskIOSaturation annotations: description: | - Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}. + Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 30 minutes, is currently at {{ printf "%.2f" $value }}. This symptom might indicate disk saturation. summary: Disk IO queue is high. expr: | @@ -443,6 +443,21 @@ labels: severity: warning {{< /code >}} +##### NodeBondingDegraded + +{{< code lang="yaml" >}} +alert: NodeBondingDegraded +annotations: + description: Bonding interface {{ $labels.master }} on {{ $labels.instance }} is + in degraded state due to one or more slave failures. + summary: Bonding interface is degraded +expr: | + (node_bonding_slaves - node_bonding_active) != 0 +for: 5m +labels: + severity: warning +{{< /code >}} + ## Recording rules {{< panel style="warning" >}} @@ -577,5 +592,6 @@ Following dashboards are generated from mixins and hosted on github: - [node-cluster-rsrc-use](https://github.com/monitoring-mixins/website/blob/master/assets/node-exporter/dashboards/node-cluster-rsrc-use.json) - [node-rsrc-use](https://github.com/monitoring-mixins/website/blob/master/assets/node-exporter/dashboards/node-rsrc-use.json) +- [nodes-aix](https://github.com/monitoring-mixins/website/blob/master/assets/node-exporter/dashboards/nodes-aix.json) - [nodes-darwin](https://github.com/monitoring-mixins/website/blob/master/assets/node-exporter/dashboards/nodes-darwin.json) - [nodes](https://github.com/monitoring-mixins/website/blob/master/assets/node-exporter/dashboards/nodes.json) diff --git a/site/content/opensearch/_index.md b/site/content/opensearch/_index.md index 2fb5ab0..2c8bb1c 100644 --- a/site/content/opensearch/_index.md +++ b/site/content/opensearch/_index.md @@ -26,7 +26,7 @@ annotations: description: '{{$labels.cluster}} health status is yellow over the last 5 minutes' summary: At least one of the clusters is reporting a yellow status. expr: | - opensearch_cluster_status{cluster!=""} == 1 + opensearch_cluster_status{opensearch_cluster!=""} == 1 for: 5m labels: severity: warning @@ -40,7 +40,7 @@ annotations: description: '{{$labels.cluster}} health status is red over the last 5 minutes' summary: At least one of the clusters is reporting a red status. expr: | - opensearch_cluster_status{cluster!=""} == 2 + opensearch_cluster_status{opensearch_cluster!=""} == 2 for: 5m labels: severity: critical @@ -55,7 +55,7 @@ annotations: {{$labels.cluster}} has had {{ printf "%.0f" $value }} shard reallocation over the last 1m which is above the threshold of 0. summary: A node has gone offline or has been disconnected triggering shard reallocation. expr: | - sum without(type) (opensearch_cluster_shards_number{cluster!="", type="relocating"}) > 0 + sum without(type) (opensearch_cluster_shards_number{opensearch_cluster!="", type="relocating"}) > 0 for: 1m labels: severity: warning @@ -70,22 +70,22 @@ annotations: {{$labels.cluster}} has had {{ printf "%.0f" $value }} shard unassigned over the last 5m which is above the threshold of 0. summary: There are shards that have been detected as unassigned. expr: | - sum without(type) (opensearch_cluster_shards_number{cluster!="", type="unassigned"}) > 0 + sum without(type) (opensearch_cluster_shards_number{opensearch_cluster!="", type="unassigned"}) > 0 for: 5m labels: severity: warning {{< /code >}} -##### OpenSearchModerateNodeDiskUsage +##### OpenSearchHighNodeDiskUsage {{< code lang="yaml" >}} -alert: OpenSearchModerateNodeDiskUsage +alert: OpenSearchHighNodeDiskUsage annotations: description: | {{$labels.node}} has had {{ printf "%.0f" $value }} disk usage over the last 5m which is above the threshold of 60. summary: The node disk usage has exceeded the warning threshold. expr: | - 100 * sum without(nodeid, path, mount, type) ((opensearch_fs_path_total_bytes{cluster!=""} - opensearch_fs_path_free_bytes{cluster!=""}) / opensearch_fs_path_total_bytes{cluster!=""}) > 60 + 100 * sum without(nodeid, path, mount, type) ((opensearch_fs_path_total_bytes{opensearch_cluster!=""} - opensearch_fs_path_free_bytes{opensearch_cluster!=""}) / opensearch_fs_path_total_bytes{opensearch_cluster!=""}) > 60 for: 5m labels: severity: warning @@ -100,22 +100,22 @@ annotations: {{$labels.node}} has had {{ printf "%.0f" $value }}% disk usage over the last 5m which is above the threshold of 80. summary: The node disk usage has exceeded the critical threshold. expr: | - 100 * sum without(nodeid, path, mount, type) ((opensearch_fs_path_total_bytes{cluster!=""} - opensearch_fs_path_free_bytes) / opensearch_fs_path_total_bytes{cluster!=""}) > 80 + 100 * sum without(nodeid, path, mount, type) ((opensearch_fs_path_total_bytes{opensearch_cluster!=""} - opensearch_fs_path_free_bytes) / opensearch_fs_path_total_bytes{opensearch_cluster!=""}) > 80 for: 5m labels: severity: critical {{< /code >}} -##### OpenSearchModerateNodeCpuUsage +##### OpenSearchHighNodeCpuUsage {{< code lang="yaml" >}} -alert: OpenSearchModerateNodeCpuUsage +alert: OpenSearchHighNodeCpuUsage annotations: description: | {{$labels.node}} has had {{ printf "%.0f" $value }}% CPU usage over the last 5m which is above the threshold of 70. summary: The node CPU usage has exceeded the warning threshold. expr: | - sum without(nodeid) (opensearch_os_cpu_percent{cluster!=""}) > 70 + sum without(nodeid) (opensearch_os_cpu_percent{opensearch_cluster!=""}) > 70 for: 5m labels: severity: warning @@ -130,22 +130,22 @@ annotations: {{$labels.node}} has had {{ printf "%.0f" $value }}% CPU usage over the last 5m which is above the threshold of 85. summary: The node CPU usage has exceeded the critical threshold. expr: | - sum without(nodeid) (opensearch_os_cpu_percent{cluster!=""}) > 85 + sum without(nodeid) (opensearch_os_cpu_percent{opensearch_cluster!=""}) > 85 for: 5m labels: severity: critical {{< /code >}} -##### OpenSearchModerateNodeMemoryUsage +##### OpenSearchHighNodeMemoryUsage {{< code lang="yaml" >}} -alert: OpenSearchModerateNodeMemoryUsage +alert: OpenSearchHighNodeMemoryUsage annotations: description: | {{$labels.node}} has had {{ printf "%.0f" $value }}% memory usage over the last 5m which is above the threshold of 70. summary: The node memory usage has exceeded the warning threshold. expr: | - sum without(nodeid) (opensearch_os_mem_used_percent{cluster!=""}) > 70 + sum without(nodeid) (opensearch_os_mem_used_percent{opensearch_cluster!=""}) > 70 for: 5m labels: severity: warning @@ -160,7 +160,7 @@ annotations: {{$labels.node}} has had {{ printf "%.0f" $value }}% memory usage over the last 5m which is above the threshold of 85. summary: The node memory usage has exceeded the critical threshold. expr: | - sum without(nodeid) (opensearch_os_mem_used_percent{cluster!=""}) > 85 + sum without(nodeid) (opensearch_os_mem_used_percent{opensearch_cluster!=""}) > 85 for: 5m labels: severity: critical @@ -175,7 +175,7 @@ annotations: {{$labels.index}} has had {{ printf "%.0f" $value }}s of request latency over the last 5m which is above the threshold of 0.5. summary: The request latency has exceeded the warning threshold. expr: | - sum without(context) ((increase(opensearch_index_search_fetch_time_seconds{cluster!="", context="total"}[5m])+increase(opensearch_index_search_query_time_seconds{context="total"}[5m])+increase(opensearch_index_search_scroll_time_seconds{context="total"}[5m])) / clamp_min(increase(opensearch_index_search_fetch_count{context="total"}[5m])+increase(opensearch_index_search_query_count{context="total"}[5m])+increase(opensearch_index_search_scroll_count{context="total"}[5m]), 1)) > 0.5 + sum without(context) ((increase(opensearch_index_search_fetch_time_seconds{opensearch_cluster!="", context="total"}[5m])+increase(opensearch_index_search_query_time_seconds{context="total"}[5m])+increase(opensearch_index_search_scroll_time_seconds{context="total"}[5m])) / clamp_min(increase(opensearch_index_search_fetch_count{context="total"}[5m])+increase(opensearch_index_search_query_count{context="total"}[5m])+increase(opensearch_index_search_scroll_count{context="total"}[5m]), 1)) > 0.5 for: 5m labels: severity: warning @@ -190,7 +190,7 @@ annotations: {{$labels.index}} has had {{ printf "%.0f" $value }}s of index latency over the last 5m which is above the threshold of 0.5. summary: The index latency has exceeded the warning threshold. expr: | - sum without(context) (increase(opensearch_index_indexing_index_time_seconds{cluster!="", context="total"}[5m]) / clamp_min(increase(opensearch_index_indexing_index_count{context="total"}[5m]), 1)) > 0.5 + sum without(context) (increase(opensearch_index_indexing_index_time_seconds{opensearch_cluster!="", context="total"}[5m]) / clamp_min(increase(opensearch_index_indexing_index_count{context="total"}[5m]), 1)) > 0.5 for: 5m labels: severity: warning diff --git a/site/content/openstack/_index.md b/site/content/openstack/_index.md index deb3a84..c91f7bb 100644 --- a/site/content/openstack/_index.md +++ b/site/content/openstack/_index.md @@ -18,37 +18,222 @@ Complete list of pregenerated alerts is available [here](https://github.com/moni ### openstack-alerts-openstack +##### OpenStackGlanceIsDown + +{{< code lang="yaml" >}} +alert: OpenStackGlanceIsDown +annotations: + description: OpenStack Glance service is down on cluster {{ $labels.instance }}. + summary: OpenStack Glance is down. +expr: | + openstack_glance_up{job=~"integrations/openstack"} == 0 +for: 5m +labels: + severity: critical +{{< /code >}} + +##### OpenStackHeatIsDown + +{{< code lang="yaml" >}} +alert: OpenStackHeatIsDown +annotations: + description: OpenStack Heat service is down on cluster {{ $labels.instance }}. + summary: OpenStack Heat is down. +expr: | + openstack_heat_up{job=~"integrations/openstack"} == 0 +for: 5m +labels: + severity: critical +{{< /code >}} + +##### OpenStackIdentityIsDown + +{{< code lang="yaml" >}} +alert: OpenStackIdentityIsDown +annotations: + description: OpenStack Identity service is down on cluster {{ $labels.instance }}. + summary: OpenStack Identity is down. +expr: | + openstack_identity_up{job=~"integrations/openstack"} == 0 +for: 5m +labels: + severity: critical +{{< /code >}} + +##### OpenStackPlacementIsDown + +{{< code lang="yaml" >}} +alert: OpenStackPlacementIsDown +annotations: + description: OpenStack Placement service is down on cluster {{ $labels.instance + }}. + summary: OpenStack Placement is down. +expr: | + openstack_placement_up{job=~"integrations/openstack"} == 0 +for: 5m +labels: + severity: critical +{{< /code >}} + ##### OpenStackPlacementHighMemoryUsageWarning {{< code lang="yaml" >}} alert: OpenStackPlacementHighMemoryUsageWarning annotations: - description: The cloud on instance {{$labels.instance}} is using {{ printf "%.0f" - $value }} percent of its allocated memory, which is above the threshold of 80 - percent. - summary: The cloud is using a significant percentage of its allocated memory. + description: | + OpenStack {{$labels.instance}} is using {{ printf "%.0f" $value }} percent of its allocated memory, + which is above the threshold of 80 percent. + summary: OpenStack is using a significant percentage of its allocated memory. expr: | - 100 * openstack_placement_resource_usage{job=~"integrations/openstack", resourcetype="MEMORY_MB"} / clamp_min(openstack_placement_resource_total{job=~"integrations/openstack", resourcetype="MEMORY_MB"}, 1) > 80 + 100 * sum by (job,instance) (openstack_placement_resource_usage{job=~"integrations/openstack", resourcetype="MEMORY_MB"}) + / + (sum by (job,instance) (openstack_placement_resource_total{job=~"integrations/openstack", resourcetype="MEMORY_MB"}) > 0) + > 80 for: 5m +keep_firing_for: 5m labels: severity: warning {{< /code >}} -##### OpenStackPlacementHighMemoryUsageCritical +##### OpenStackNovaAgentDown {{< code lang="yaml" >}} -alert: OpenStackPlacementHighMemoryUsageCritical +alert: OpenStackNovaAgentDown annotations: - description: The cloud on instance {{$labels.instance}} is using {{ printf "%.0f" - $value }} percent of its allocated memory, which is above the threshold of 90 - percent. - summary: The cloud is using a large percentage of its allocated memory, consider + description: | + OpenStack {{$labels.instance}} is using {{ printf "%.0f" $value }} percent of its allocated memory, + which is above the threshold of 90 percent. + summary: OpenStack is using a large percentage of its allocated memory, consider allocating more resources. -expr: "100 * openstack_placement_resource_usage{job=~\"integrations/openstack\", resourcetype=\"MEMORY_MB\"} - / clamp_min(openstack_placement_resource_total{job=~\"integrations/openstack\", - resourcetype=\"MEMORY_MB\"}, 1) > 90 +expr: | + 100 * sum by (job,instance) (openstack_placement_resource_usage{job=~"integrations/openstack", resourcetype="MEMORY_MB"}) + / + (sum by (job,instance) (openstack_placement_resource_total{job=~"integrations/openstack", resourcetype="MEMORY_MB"}) > 0) + > 90 +for: 5m +keep_firing_for: 5m +labels: + severity: critical +{{< /code >}} + +##### OpenStackPlacementHighVCPUUsageWarning + +{{< code lang="yaml" >}} +alert: OpenStackPlacementHighVCPUUsageWarning +annotations: + description: | + OpenStack {{$labels.instance}} is using {{ printf "%.0f" $value }} percent of its allocated vCPU, + which is above the threshold of 80 percent. + summary: OpenStack is using a significant percentage of its allocated vCPU. +expr: | + 100 * sum by (job,instance) (openstack_placement_resource_usage{job=~"integrations/openstack", resourcetype="VCPU"}) + / + (sum by (job,instance) (openstack_placement_resource_total{job=~"integrations/openstack", resourcetype="VCPU"}) > 0) + > 80 +for: 5m +keep_firing_for: 5m +labels: + severity: warning +{{< /code >}} + +##### OpenStackPlacementHighVCPUUsageCritical + +{{< code lang="yaml" >}} +alert: OpenStackPlacementHighVCPUUsageCritical +annotations: + description: | + OpenStack {{$labels.instance}} is using {{ printf "%.0f" $value }} percent of its allocated vCPU, + which is above the threshold of 90 percent. + summary: OpenStack is using a large percentage of its allocated vCPU, consider allocating + more resources. +expr: | + 100 * sum by (job,instance) (openstack_placement_resource_usage{job=~"integrations/openstack", resourcetype="VCPU"}) + / + (sum by (job,instance) (openstack_placement_resource_total{job=~"integrations/openstack", resourcetype="VCPU"}) > 0) + > 90 +for: 5m +keep_firing_for: 5m +labels: + severity: critical +{{< /code >}} + +##### OpenStackNeutronHighIPsUsageWarning + +{{< code lang="yaml" >}} +alert: OpenStackNeutronHighIPsUsageWarning +annotations: + description: | + Network {{$labels.network_name}} is running out of free IP addresses on OpenStack {{$labels.instance}}, + {{ printf "%.0f" $value }} percent of the pool used, + {{ with printf `sum(openstack_neutron_network_ip_availabilities_total{job=~"integrations/openstack", instance=~"%s", network_name=~"%s"}) - (sum(openstack_neutron_network_ip_availabilities_used{job=~"integrations/openstack", instance=~"%s", network_name=~"%s"}))` .Labels.instance .Labels.network_name .Labels.instance .Labels.network_name | query -}}{{ . | first | value | humanize }}{{ end }} IP addresses available. + summary: Free IP addresses are running out. +expr: "100 * +sum by (job,instance, network_name) (openstack_neutron_network_ip_availabilities_used{job=~\"integrations/openstack\", + network_name=~\".+\"}) +/ +(sum by (job,instance, network_name) (openstack_neutron_network_ip_availabilities_total{job=~\"integrations/openstack\", + network_name=~\".+\"}) +> 0) +> 80 " for: 5m +keep_firing_for: 5m +labels: + severity: warning +{{< /code >}} + +##### OpenStackNeutronHighIPsUsageCritical + +{{< code lang="yaml" >}} +alert: OpenStackNeutronHighIPsUsageCritical +annotations: + description: | + Network {{$labels.network_name}} is running out of free IP addresses on OpenStack {{$labels.instance}}, + {{ printf "%.0f" $value }} percent of the pool used, + {{ with printf `sum(openstack_neutron_network_ip_availabilities_total{job=~"integrations/openstack", instance=~"%s", network_name=~"%s"}) - (sum(openstack_neutron_network_ip_availabilities_used{job=~"integrations/openstack", instance=~"%s", network_name=~"%s"}))` .Labels.instance .Labels.network_name .Labels.instance .Labels.network_name | query -}}{{ . | first | value | humanize }}{{ end }} IP addresses available. + summary: There are practically no free IP addresses left. +expr: "100 * +sum by (job,instance, network_name) (openstack_neutron_network_ip_availabilities_used{job=~\"integrations/openstack\", + network_name=~\".+\"}) +/ +(sum by (job,instance, network_name) (openstack_neutron_network_ip_availabilities_total{job=~\"integrations/openstack\", + network_name=~\".+\"}) +> 0) +> 90 +" +for: 5m +keep_firing_for: 5m +labels: + severity: critical +{{< /code >}} + +### openstack-nova-alertsopenstack + +##### OpenStackNovaIsDown + +{{< code lang="yaml" >}} +alert: OpenStackNovaIsDown +annotations: + description: OpenStack Nova is down on {{ $labels.instance }}. + summary: OpenStack Nova service is down. +expr: | + openstack_nova_up{job=~"integrations/openstack"} == 0 +for: 5m +labels: + severity: critical +{{< /code >}} + +##### OpenStackNovaAgentIsDown + +{{< code lang="yaml" >}} +alert: OpenStackNovaAgentIsDown +annotations: + description: An OpenStack Nova agent is down on hostname {{ $labels.hostname }} + on OpenStack cluster {{ $labels.instance }}. + summary: OpenStack Nova agent is down on the specific node. +expr: | + openstack_nova_agent_state{job=~"integrations/openstack",adminState="enabled"} != 1 +for: 5m labels: severity: critical {{< /code >}} @@ -58,12 +243,12 @@ labels: {{< code lang="yaml" >}} alert: OpenStackNovaHighVMMemoryUsage annotations: - description: Virtual machines on the cloud on {{$labels.instance}} are using {{ - printf "%.0f" $value }} percent of their allocated memory, which is above the - threshold of 80 percent. + description: | + Virtual machines on OpenStack {{ $labels.instance }} are using {{ printf "%.0f" $value }} percent of their allocated memory, + which is above the threshold of 80 percent. summary: VMs are using a high percentage of their allocated memory. expr: | - 100 * openstack_nova_limits_memory_used{job=~"integrations/openstack"} / clamp_min(openstack_nova_limits_memory_max{job=~"integrations/openstack"}, 1) > 80 + 100 * openstack_nova_limits_memory_used{job=~"integrations/openstack"} / (openstack_nova_limits_memory_max{job=~"integrations/openstack"} > 0) > 80 for: 5m labels: severity: warning @@ -74,25 +259,74 @@ labels: {{< code lang="yaml" >}} alert: OpenStackNovaHighVMVCPUUsage annotations: - description: Virtual machines on the cloud on {{$labels.instance}} are using {{ - printf "%.0f" $value }} percent of their allocated virtual CPUs, which is above - the threshold of 80 percent. + description: | + Virtual machines on OpenStack {{$labels.instance}} are using {{ printf "%.0f" $value }} percent of their allocated virtual CPUs, + which is above the threshold of 80 percent. summary: VMs are using a high percentage of their allocated virtual CPUs. expr: | - 100 * openstack_nova_limits_vcpus_used{job=~"integrations/openstack"} / clamp_min(openstack_nova_limits_vcpus_max{job=~"integrations/openstack"}, 1) > 80 + 100 * openstack_nova_limits_vcpus_used{job=~"integrations/openstack"} / (openstack_nova_limits_vcpus_max{job=~"integrations/openstack"} > 0) > 80 for: 5m labels: severity: warning {{< /code >}} +### openstack-neutron-alertsopenstack + +##### OpenStackNeutronIsDown + +{{< code lang="yaml" >}} +alert: OpenStackNeutronIsDown +annotations: + description: OpenStack Neutron service is down on cluster {{ $labels.instance }}. + summary: OpenStack Neutron is down. +expr: | + openstack_neutron_up{job=~"integrations/openstack"} == 0 +for: 5m +labels: + severity: critical +{{< /code >}} + +##### OpenStackNeutronAgentIsDown + +{{< code lang="yaml" >}} +alert: OpenStackNeutronAgentIsDown +annotations: + description: | + OpenStack Neutron agent`s service {{ $labels.service }} is down on hostname {{ $labels.hostname }} on OpenStack cluster {{ $labels.instance }}. + If {{ $labels.service }} is no longer required on this host, disable it administratively by running: + OpenStack network agent set {{ $labels.id }} --disable + runbook_url: https://docs.openstack.org/neutron/zed/admin/config-services-agent.html#agent-s-admin-state-specific-config-options + summary: OpenStack Neutron agent is down on the specific node. +expr: | + openstack_neutron_agent_state{job=~"integrations/openstack",adminState="up"} != 1 +for: 5m +labels: + severity: critical +{{< /code >}} + +##### OpenStackNeutronL3AgentIsDown + +{{< code lang="yaml" >}} +alert: OpenStackNeutronL3AgentIsDown +annotations: + description: OpenStack Neutron L3 agent is down on hostname {{ $labels.agent_host + }} on OpenStack cluster {{ $labels.instance }}. + summary: OpenStack Neutron L3 agent is down on the specific node. +expr: | + openstack_neutron_l3_agent_of_router{job=~"integrations/openstack",agent_admin_up="true"} != 1 +for: 5m +labels: + severity: critical +{{< /code >}} + ##### OpenStackNeutronHighDisconnectedPortRate {{< code lang="yaml" >}} alert: OpenStackNeutronHighDisconnectedPortRate annotations: - description: '{{ printf "%.0f" $value }} percent of ports managed by the Neutron - service on instance {{$labels.instance}} have no IP addresses assigned to them, - which is above the threshold of 25' + description: | + {{ printf "%.0f" $value }} percent of ports managed by the Neutron service on OpenStack cluster {{$labels.instance}} have no IP addresses assigned to them, + which is above the threshold of 25. summary: A high rate of ports have no IP addresses assigned to them. expr: | 100 * openstack_neutron_ports_no_ips{job=~"integrations/openstack"} / clamp_min(openstack_neutron_ports{job=~"integrations/openstack"}, 1) > 25 @@ -106,9 +340,9 @@ labels: {{< code lang="yaml" >}} alert: OpenStackNeutronHighInactiveRouterRate annotations: - description: '{{ printf "%.0f" $value }} percent of routers managed by the Neutron - service on instance {{$labels.instance}} are currently inactive, which is above - the threshold of 15' + description: | + {{ printf "%.0f" $value }} percent of routers managed by the Neutron service on cluster {{$labels.instance}} are currently inactive, + which is above the threshold of 15. summary: A high rate of routers are currently inactive. expr: | 100 * openstack_neutron_routers_not_active{job=~"integrations/openstack"} / clamp_min(openstack_neutron_routers{job=~"integrations/openstack"}, 1) > 15 @@ -117,18 +351,49 @@ labels: severity: critical {{< /code >}} -##### OpenStackCinderHighBackupMemoryUsage +### openstack-cinder-alertsopenstack + +##### OpenStackCinderIsDown {{< code lang="yaml" >}} -alert: OpenStackCinderHighBackupMemoryUsage +alert: OpenStackCinderIsDown annotations: - description: Backups managed by the Cinder service on instance {{$labels.instance}} - are using {{ printf "%.0f" $value }} percent of their allocated memory, which - is above the threshold of 80 percent. - summary: Cinder backups are using a large amount of their maximum memory. + description: OpenStack Cinder service is down on cluster {{ $labels.instance }}. + summary: OpenStack Cinder is down. expr: | - 100 * openstack_cinder_limits_backup_used_gb{job=~"integrations/openstack"} / clamp_min(openstack_cinder_limits_backup_max_gb{job=~"integrations/openstack"}, 1) > 80 + openstack_cinder_up{job=~"integrations/openstack"} == 0 for: 5m +labels: + severity: critical +{{< /code >}} + +##### OpenStackCinderAgentIsDown + +{{< code lang="yaml" >}} +alert: OpenStackCinderAgentIsDown +annotations: + description: OpenStack Cinder agent is down on hostname {{ $labels.hostname }} on + OpenStack cluster {{ $labels.instance }}. + summary: OpenStack Cinder agent is down on the specific node. +expr: | + openstack_cinder_agent_state{job=~"integrations/openstack",adminState="enabled"} != 1 +for: 5m +labels: + severity: critical +{{< /code >}} + +##### OpenStackCinderHighPoolCapacityUsage + +{{< code lang="yaml" >}} +alert: OpenStackCinderHighPoolCapacityUsage +annotations: + description: | + Pools managed by the Cinder service on cluster {{$labels.instance}} are using {{ printf "%.0f" $value }} percent of their allocated capacity, + which is above the threshold of 80 percent. + summary: Cinder pools are using a large amount of their maximum capacity. +expr: | + 100 * (openstack_cinder_pool_capacity_total_gb{job=~"integrations/openstack"} - openstack_cinder_pool_capacity_free_gb{job=~"integrations/openstack"}) / clamp_min(openstack_cinder_pool_capacity_total_gb{job=~"integrations/openstack"}, 1) > 80 +for: 10m labels: severity: warning {{< /code >}} @@ -138,28 +403,28 @@ labels: {{< code lang="yaml" >}} alert: OpenStackCinderHighVolumeMemoryUsage annotations: - description: Volumes managed by the Cinder service on instance {{$labels.instance}} - are using {{ printf "%.0f" $value }} percent of their allocated memory, which - is above the threshold of 80 percent. + description: | + Volumes managed by the Cinder service on cluster {{$labels.instance}} are using {{ printf "%.0f" $value }} percent of their allocated memory, + which is above the threshold of 80 percent. summary: Cinder volumes are using a large amount of their maximum memory. expr: | - 100 * openstack_cinder_limits_volume_used_gb{job=~"integrations/openstack"} / clamp_min(openstack_cinder_limits_volume_max_gb{job=~"integrations/openstack"}, 1) > 80 + 100 * openstack_cinder_limits_volume_used_gb{job=~"integrations/openstack"} / (openstack_cinder_limits_volume_max_gb{job=~"integrations/openstack"} > 0) > 80 for: 5m labels: severity: warning {{< /code >}} -##### OpenStackCinderHighPoolCapacityUsage +##### OpenStackCinderHighBackupMemoryUsage {{< code lang="yaml" >}} -alert: OpenStackCinderHighPoolCapacityUsage +alert: OpenStackCinderHighBackupMemoryUsage annotations: - description: Pools managed by the Cinder service on instance {{$labels.instance}} - are using {{ printf "%.0f" $value }} percent of their allocated capacity, which - is above the threshold of 80 percent. - summary: Cinder pools are using a large amount of their maximum capacity. + description: | + Backups managed by the Cinder service on cluster {{$labels.instance}} are using {{ printf "%.0f" $value }} percent of their allocated memory, + which is above the threshold of 80 percent. + summary: Cinder backups are using a large amount of their maximum memory. expr: | - 100 * (openstack_cinder_pool_capacity_total_gb{job=~"integrations/openstack"} - openstack_cinder_pool_capacity_free_gb{job=~"integrations/openstack"}) / clamp_min(openstack_cinder_pool_capacity_total_gb{job=~"integrations/openstack"}, 1) > 80 + 100 * openstack_cinder_limits_backup_used_gb{job=~"integrations/openstack"} / (openstack_cinder_limits_backup_max_gb{job=~"integrations/openstack"} > 0) > 80 for: 5m labels: severity: warning diff --git a/site/content/prometheus-operator/_index.md b/site/content/prometheus-operator/_index.md index 80dce80..759abe2 100644 --- a/site/content/prometheus-operator/_index.md +++ b/site/content/prometheus-operator/_index.md @@ -70,7 +70,7 @@ alert: PrometheusOperatorReconcileErrors annotations: description: '{{ $value | humanizePercentage }} of reconciling operations failed for {{ $labels.controller }} controller in {{ $labels.namespace }} namespace.' - summary: Errors while reconciling controller. + summary: Errors while reconciling objects. expr: | (sum by (controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="prometheus-operator"}[5m]))) / (sum by (controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="prometheus-operator"}[5m]))) > 0.1 for: 10m @@ -78,6 +78,21 @@ labels: severity: warning {{< /code >}} +##### PrometheusOperatorStatusUpdateErrors + +{{< code lang="yaml" >}} +alert: PrometheusOperatorStatusUpdateErrors +annotations: + description: '{{ $value | humanizePercentage }} of status update operations failed + for {{ $labels.controller }} controller in {{ $labels.namespace }} namespace.' + summary: Errors while updating objects status. +expr: | + (sum by (controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="prometheus-operator"}[5m]))) / (sum by (controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="prometheus-operator"}[5m]))) > 0.1 +for: 10m +labels: + severity: warning +{{< /code >}} + ##### PrometheusOperatorNodeLookupErrors {{< code lang="yaml" >}} diff --git a/site/content/prometheus/_index.md b/site/content/prometheus/_index.md index 17157f9..00d41fb 100644 --- a/site/content/prometheus/_index.md +++ b/site/content/prometheus/_index.md @@ -49,6 +49,22 @@ labels: severity: warning {{< /code >}} +##### PrometheusKubernetesListWatchFailures + +{{< code lang="yaml" >}} +alert: PrometheusKubernetesListWatchFailures +annotations: + description: Kubernetes service discovery of Prometheus {{$labels.instance}} is + experiencing {{ printf "%.0f" $value }} failures with LIST/WATCH requests to the + Kubernetes API in the last 5 minutes. + summary: Requests in Kubernetes SD are failing. +expr: | + increase(prometheus_sd_kubernetes_failures_total{job="prometheus"}[5m]) > 0 +for: 15m +labels: + severity: warning +{{< /code >}} + ##### PrometheusNotificationQueueRunningFull Prometheus alert notification queue predicted to run full in less than @@ -152,7 +168,7 @@ annotations: summary: Prometheus is not ingesting samples. expr: | ( - rate(prometheus_tsdb_head_samples_appended_total{job="prometheus"}[5m]) <= 0 + sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="prometheus"}[5m])) <= 0 and ( sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="prometheus"}) > 0 diff --git a/site/content/promtail/_index.md b/site/content/promtail/_index.md index 3a9a387..26de616 100644 --- a/site/content/promtail/_index.md +++ b/site/content/promtail/_index.md @@ -23,8 +23,9 @@ Complete list of pregenerated alerts is available [here](https://github.com/moni {{< code lang="yaml" >}} alert: PromtailRequestsErrors annotations: - message: | + description: | {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors. + summary: Promtail request error rate is high. expr: | 100 * sum(rate(promtail_request_duration_seconds_count{status_code=~"5..|failed"}[1m])) by (namespace, job, route, instance) / @@ -40,8 +41,9 @@ labels: {{< code lang="yaml" >}} alert: PromtailRequestLatency annotations: - message: | + description: | {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency. + summary: Promtail request latency P99 is high. expr: | job_status_code_namespace:promtail_request_duration_seconds:99quantile > 1 for: 15m @@ -54,8 +56,9 @@ labels: {{< code lang="yaml" >}} alert: PromtailFileMissing annotations: - message: | + description: | {{ $labels.instance }} {{ $labels.job }} {{ $labels.path }} matches the glob but is not being tailed. + summary: Promtail cannot find a file it should be tailing. expr: | promtail_file_bytes_total unless promtail_read_bytes_total for: 15m diff --git a/site/content/snmp/_index.md b/site/content/snmp/_index.md index 339658d..e7605be 100644 --- a/site/content/snmp/_index.md +++ b/site/content/snmp/_index.md @@ -46,6 +46,20 @@ labels: severity: warning {{< /code >}} +##### SNMPRestarted + +{{< code lang="yaml" >}} +alert: SNMPRestarted +annotations: + description: The SNMP target on instance {{$labels.instance}} from job {{$labels.job}} + has restarted. + summary: SNMP target restarted. +expr: sysUpTime < 300 +for: 5s +labels: + severity: info +{{< /code >}} + ## Dashboards Following dashboards are generated from mixins and hosted on github: diff --git a/site/content/spring-boot/_index.md b/site/content/spring-boot/_index.md index 3cd6236..7b583dc 100644 --- a/site/content/spring-boot/_index.md +++ b/site/content/spring-boot/_index.md @@ -10,8 +10,3 @@ title: spring-boot Jsonnet source code is available at [github.com/grafana/jsonnet-libs](https://github.com/grafana/jsonnet-libs/tree/master/spring-boot-mixin) {{< /panel >}} -## Dashboards -Following dashboards are generated from mixins and hosted on github: - - -- [spring-boot-dashboard](https://github.com/monitoring-mixins/website/blob/master/assets/spring-boot/dashboards/spring-boot-dashboard.json) diff --git a/site/content/thanos/_index.md b/site/content/thanos/_index.md index d4715bb..bf8dd8b 100644 --- a/site/content/thanos/_index.md +++ b/site/content/thanos/_index.md @@ -265,9 +265,9 @@ https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanos alert: ThanosQueryOverload annotations: description: Thanos Query {{$labels.job}} has been overloaded for more than 15 minutes. - This may be a symptom of excessive simultanous complex requests, low performance + This may be a symptom of excessive simultaneous complex requests, low performance of the Prometheus API, or failures within these components. Assess the health - of the Thanos query instances, the connnected Prometheus instances, look for potential + of the Thanos query instances, the connected Prometheus instances, look for potential senders of these requests and then contact support. runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosqueryoverload summary: Thanos query reaches its maximum capacity serving concurrent requests. @@ -1043,7 +1043,7 @@ record: :api_range_query_duration_seconds:histogram_quantile ### thanos-receive.rules -##### :grpc_server_failures_per_unary:sum_rate +##### :thanos_query_receive_grpc_server_failures_per_unary:sum_rate {{< code lang="yaml" >}} expr: | @@ -1052,10 +1052,10 @@ expr: | / sum by (job) (rate(grpc_server_started_total{job=~".*thanos-receive.*", grpc_type="unary"}[5m])) ) -record: :grpc_server_failures_per_unary:sum_rate +record: :thanos_query_receive_grpc_server_failures_per_unary:sum_rate {{< /code >}} -##### :grpc_server_failures_per_stream:sum_rate +##### :thanos_query_receive_grpc_server_failures_per_stream:sum_rate {{< code lang="yaml" >}} expr: | @@ -1064,7 +1064,7 @@ expr: | / sum by (job) (rate(grpc_server_started_total{job=~".*thanos-receive.*", grpc_type="server_stream"}[5m])) ) -record: :grpc_server_failures_per_stream:sum_rate +record: :thanos_query_receive_grpc_server_failures_per_stream:sum_rate {{< /code >}} ##### :http_failure_per_request:sum_rate @@ -1129,7 +1129,7 @@ record: :thanos_receive_hashring_file_failure_per_refresh:sum_rate ### thanos-store.rules -##### :grpc_server_failures_per_unary:sum_rate +##### :thanos_query_store_grpc_server_failures_per_unary:sum_rate {{< code lang="yaml" >}} expr: | @@ -1138,10 +1138,10 @@ expr: | / sum by (job) (rate(grpc_server_started_total{job=~".*thanos-store.*", grpc_type="unary"}[5m])) ) -record: :grpc_server_failures_per_unary:sum_rate +record: :thanos_query_store_grpc_server_failures_per_unary:sum_rate {{< /code >}} -##### :grpc_server_failures_per_stream:sum_rate +##### :thanos_query_store_grpc_server_failures_per_stream:sum_rate {{< code lang="yaml" >}} expr: | @@ -1150,7 +1150,7 @@ expr: | / sum by (job) (rate(grpc_server_started_total{job=~".*thanos-store.*", grpc_type="server_stream"}[5m])) ) -record: :grpc_server_failures_per_stream:sum_rate +record: :thanos_query_store_grpc_server_failures_per_stream:sum_rate {{< /code >}} ##### :thanos_objstore_bucket_failures_per_operation:sum_rate diff --git a/site/content/windows-active-directory/_index.md b/site/content/windows-active-directory/_index.md index 53982d5..4c0a8e2 100644 --- a/site/content/windows-active-directory/_index.md +++ b/site/content/windows-active-directory/_index.md @@ -29,6 +29,7 @@ annotations: expr: | 100 - (avg without (mode, core) (rate(windows_cpu_time_total{job=~"integrations/windows_exporter", mode="idle"}[2m])) * 100) > 90 for: 15m +keep_firing_for: 5m labels: severity: warning {{< /code >}} @@ -46,6 +47,7 @@ expr: | / windows_cs_physical_memory_bytes{job=~"integrations/windows_exporter"}) * 100) > 90 for: 15m +keep_firing_for: 5m labels: severity: critical {{< /code >}} @@ -61,6 +63,7 @@ annotations: expr: | 100 - ((windows_logical_disk_free_bytes{job=~"integrations/windows_exporter"} ) / (windows_logical_disk_size_bytes{job=~"integrations/windows_exporter"})) * 100 > 90 for: 15m +keep_firing_for: 5m labels: severity: critical {{< /code >}} @@ -101,11 +104,12 @@ labels: alert: WindowsNTPClientDelay annotations: description: | - 'Round-trip time of NTP client on instance {{ $labels.instance }} is greater than 1 second. Delay is {{ $value }} sec.' + Round-trip time of NTP client on instance {{ $labels.instance }} is greater than 1 second. Delay is {{ $value }} sec. summary: NTP client delay. expr: | windows_time_ntp_round_trip_delay_seconds{job=~"integrations/windows_exporter"} > 1 for: 5m +keep_firing_for: 5m labels: severity: warning {{< /code >}} @@ -116,11 +120,12 @@ labels: alert: WindowsNTPTimeOffset annotations: description: | - 'NTP time offset for instance {{ $labels.instance }} is greater than 1 second. Offset is {{ $value }} sec.' + NTP time offset for instance {{ $labels.instance }} is greater than 1 second. Offset is {{ $value }} sec. summary: NTP time offset is too large. expr: | windows_time_computed_time_offset_seconds{job=~"integrations/windows_exporter"} > 1 for: 5m +keep_firing_for: 5m labels: severity: warning {{< /code >}} @@ -139,6 +144,7 @@ expr: "windows_ad_replication_pending_operations{job=~\"integrations/windows_exp >= 50 " for: 10m +keep_firing_for: 5m labels: severity: warning {{< /code >}} @@ -157,6 +163,7 @@ expr: "increase(windows_ad_replication_sync_requests_schema_mismatch_failure_tot > 0 " for: 5m +keep_firing_for: 5m labels: severity: critical {{< /code >}} @@ -167,13 +174,15 @@ labels: alert: WindowsActiveDirectoryHighPasswordChanges annotations: description: The number of password changes on {{$labels.instance}} is {{ printf - "%.0f" $value }} which is greater than the threshold of 25 + "%.0f" $value }} which is greater than the threshold of 25. This alert would resolve + itself if no new anomalies are detected within 24 hours. summary: There is a high number of password changes. This may indicate unauthorized changes or attacks. expr: | increase(windows_ad_sam_password_changes_total{job=~"integrations/windows_exporter"}[5m]) > 25 for: 5m labels: + keep_firing_for: 24h severity: warning {{< /code >}} diff --git a/site/content/windows/_index.md b/site/content/windows/_index.md index 65c2e55..2b4d094 100644 --- a/site/content/windows/_index.md +++ b/site/content/windows/_index.md @@ -29,6 +29,7 @@ annotations: expr: | 100 - (avg without (mode, core) (rate(windows_cpu_time_total{job=~".*windows.*", mode="idle"}[2m])) * 100) > 90 for: 15m +keep_firing_for: 5m labels: severity: warning {{< /code >}} @@ -46,6 +47,7 @@ expr: | / windows_cs_physical_memory_bytes{job=~".*windows.*"}) * 100) > 90 for: 15m +keep_firing_for: 5m labels: severity: critical {{< /code >}} @@ -61,6 +63,7 @@ annotations: expr: | 100 - ((windows_logical_disk_free_bytes{job=~".*windows.*"} ) / (windows_logical_disk_size_bytes{job=~".*windows.*"})) * 100 > 90 for: 15m +keep_firing_for: 5m labels: severity: critical {{< /code >}} @@ -101,11 +104,12 @@ labels: alert: WindowsNTPClientDelay annotations: description: | - 'Round-trip time of NTP client on instance {{ $labels.instance }} is greater than 1 second. Delay is {{ $value }} sec.' + Round-trip time of NTP client on instance {{ $labels.instance }} is greater than 1 second. Delay is {{ $value }} sec. summary: NTP client delay. expr: | windows_time_ntp_round_trip_delay_seconds{job=~".*windows.*"} > 1 for: 5m +keep_firing_for: 5m labels: severity: warning {{< /code >}} @@ -116,11 +120,12 @@ labels: alert: WindowsNTPTimeOffset annotations: description: | - 'NTP time offset for instance {{ $labels.instance }} is greater than 1 second. Offset is {{ $value }} sec.' + NTP time offset for instance {{ $labels.instance }} is greater than 1 second. Offset is {{ $value }} sec. summary: NTP time offset is too large. expr: | windows_time_computed_time_offset_seconds{job=~".*windows.*"} > 1 for: 5m +keep_firing_for: 5m labels: severity: warning {{< /code >}}