assets,site/content: daily assets regeneration

assets/etcd/alerts.yaml

@@ -150,7 +150,7 @@ groups:
         quota as the writes to etcd will be disabled when it is full.'
       summary: etcd cluster database is running full.
     expr: |
-      (etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100 > 95
+      (last_over_time(etcd_mvcc_db_total_size_in_bytes[5m]) / last_over_time(etcd_server_quota_backend_bytes[5m]))*100 > 95
     for: 10m
     labels:
       severity: critical
@@ -166,3 +166,17 @@ groups:
     for: 10m
     labels:
       severity: warning
+  - alert: etcdDatabaseHighFragmentationRatio
+    annotations:
+      description: 'etcd cluster "{{ $labels.job }}": database size in use on instance
+        {{ $labels.instance }} is {{ $value | humanizePercentage }} of the actual
+        allocated disk space, please run defragmentation (e.g. etcdctl defrag) to
+        retrieve the unused fragmented disk space.'
+      runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation
+      summary: etcd database size in use is less than 50% of the actual allocated
+        storage.
+    expr: |
+      (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes[5m])) < 0.5
+    for: 10m
+    labels:
+      severity: warning

assets/prometheus/alerts.yaml

@@ -250,7 +250,7 @@ groups:
         capacity in its query engine for the last 15 minutes.
       summary: Prometheus is reaching its maximum capacity serving concurrent requests.
     expr: |
-      avg_over_time(prometheus_engine_queries{job="prometheus-k8s"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="prometheus-k8s"}[5m]) > 0.8
+      avg_over_time(prometheus_engine_queries{job="prometheus"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="prometheus"}[5m]) > 0.8
     for: 15m
     labels:
       severity: warning

site/content/etcd/_index.md

@@ -227,7 +227,7 @@ annotations:
     as the writes to etcd will be disabled when it is full.'
   summary: etcd cluster database is running full.
 expr: |
-  (etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100 > 95
+  (last_over_time(etcd_mvcc_db_total_size_in_bytes[5m]) / last_over_time(etcd_server_quota_backend_bytes[5m]))*100 > 95
 for: 10m
 labels:
   severity: critical
@@ -249,6 +249,24 @@ labels:
   severity: warning
 {{< /code >}}
  
+##### etcdDatabaseHighFragmentationRatio
+
+{{< code lang="yaml" >}}
+alert: etcdDatabaseHighFragmentationRatio
+annotations:
+  description: 'etcd cluster "{{ $labels.job }}": database size in use on instance
+    {{ $labels.instance }} is {{ $value | humanizePercentage }} of the actual allocated
+    disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused
+    fragmented disk space.'
+  runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation
+  summary: etcd database size in use is less than 50% of the actual allocated storage.
+expr: |
+  (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes[5m])) < 0.5
+for: 10m
+labels:
+  severity: warning
+{{< /code >}}
+ 
 ## Dashboards
 Following dashboards are generated from mixins and hosted on github:
 

site/content/prometheus/_index.md

@@ -369,7 +369,7 @@ annotations:
     capacity in its query engine for the last 15 minutes.
   summary: Prometheus is reaching its maximum capacity serving concurrent requests.
 expr: |
-  avg_over_time(prometheus_engine_queries{job="prometheus-k8s"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="prometheus-k8s"}[5m]) > 0.8
+  avg_over_time(prometheus_engine_queries{job="prometheus"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="prometheus"}[5m]) > 0.8
 for: 15m
 labels:
   severity: warning