mirror of
https://github.com/monitoring-mixins/website.git
synced 2024-12-14 11:37:31 +00:00
308 lines
12 KiB
YAML
308 lines
12 KiB
YAML
groups:
|
|
- name: node-exporter
|
|
rules:
|
|
- alert: NodeFilesystemSpaceFillingUp
|
|
annotations:
|
|
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
|
|
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
|
|
space left and is filling up.
|
|
summary: Filesystem is predicted to run out of space within the next 24 hours.
|
|
expr: |
|
|
(
|
|
node_filesystem_avail_bytes{job="node",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node",fstype!="",mountpoint!=""} * 100 < 40
|
|
and
|
|
predict_linear(node_filesystem_avail_bytes{job="node",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0
|
|
and
|
|
node_filesystem_readonly{job="node",fstype!="",mountpoint!=""} == 0
|
|
)
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeFilesystemSpaceFillingUp
|
|
annotations:
|
|
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
|
|
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
|
|
space left and is filling up fast.
|
|
summary: Filesystem is predicted to run out of space within the next 4 hours.
|
|
expr: |
|
|
(
|
|
node_filesystem_avail_bytes{job="node",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node",fstype!="",mountpoint!=""} * 100 < 20
|
|
and
|
|
predict_linear(node_filesystem_avail_bytes{job="node",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0
|
|
and
|
|
node_filesystem_readonly{job="node",fstype!="",mountpoint!=""} == 0
|
|
)
|
|
for: 1h
|
|
labels:
|
|
severity: critical
|
|
- alert: NodeFilesystemAlmostOutOfSpace
|
|
annotations:
|
|
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
|
|
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
|
|
space left.
|
|
summary: Filesystem has less than 5% space left.
|
|
expr: |
|
|
(
|
|
node_filesystem_avail_bytes{job="node",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node",fstype!="",mountpoint!=""} * 100 < 5
|
|
and
|
|
node_filesystem_readonly{job="node",fstype!="",mountpoint!=""} == 0
|
|
)
|
|
for: 30m
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeFilesystemAlmostOutOfSpace
|
|
annotations:
|
|
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
|
|
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
|
|
space left.
|
|
summary: Filesystem has less than 3% space left.
|
|
expr: |
|
|
(
|
|
node_filesystem_avail_bytes{job="node",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node",fstype!="",mountpoint!=""} * 100 < 3
|
|
and
|
|
node_filesystem_readonly{job="node",fstype!="",mountpoint!=""} == 0
|
|
)
|
|
for: 30m
|
|
labels:
|
|
severity: critical
|
|
- alert: NodeFilesystemFilesFillingUp
|
|
annotations:
|
|
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
|
|
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
|
|
inodes left and is filling up.
|
|
summary: Filesystem is predicted to run out of inodes within the next 24 hours.
|
|
expr: |
|
|
(
|
|
node_filesystem_files_free{job="node",fstype!="",mountpoint!=""} / node_filesystem_files{job="node",fstype!="",mountpoint!=""} * 100 < 40
|
|
and
|
|
predict_linear(node_filesystem_files_free{job="node",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0
|
|
and
|
|
node_filesystem_readonly{job="node",fstype!="",mountpoint!=""} == 0
|
|
)
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeFilesystemFilesFillingUp
|
|
annotations:
|
|
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
|
|
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
|
|
inodes left and is filling up fast.
|
|
summary: Filesystem is predicted to run out of inodes within the next 4 hours.
|
|
expr: |
|
|
(
|
|
node_filesystem_files_free{job="node",fstype!="",mountpoint!=""} / node_filesystem_files{job="node",fstype!="",mountpoint!=""} * 100 < 20
|
|
and
|
|
predict_linear(node_filesystem_files_free{job="node",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0
|
|
and
|
|
node_filesystem_readonly{job="node",fstype!="",mountpoint!=""} == 0
|
|
)
|
|
for: 1h
|
|
labels:
|
|
severity: critical
|
|
- alert: NodeFilesystemAlmostOutOfFiles
|
|
annotations:
|
|
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
|
|
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
|
|
inodes left.
|
|
summary: Filesystem has less than 5% inodes left.
|
|
expr: |
|
|
(
|
|
node_filesystem_files_free{job="node",fstype!="",mountpoint!=""} / node_filesystem_files{job="node",fstype!="",mountpoint!=""} * 100 < 5
|
|
and
|
|
node_filesystem_readonly{job="node",fstype!="",mountpoint!=""} == 0
|
|
)
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeFilesystemAlmostOutOfFiles
|
|
annotations:
|
|
description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint
|
|
}}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available
|
|
inodes left.
|
|
summary: Filesystem has less than 3% inodes left.
|
|
expr: |
|
|
(
|
|
node_filesystem_files_free{job="node",fstype!="",mountpoint!=""} / node_filesystem_files{job="node",fstype!="",mountpoint!=""} * 100 < 3
|
|
and
|
|
node_filesystem_readonly{job="node",fstype!="",mountpoint!=""} == 0
|
|
)
|
|
for: 1h
|
|
labels:
|
|
severity: critical
|
|
- alert: NodeNetworkReceiveErrs
|
|
annotations:
|
|
description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
|
|
{{ printf "%.0f" $value }} receive errors in the last two minutes.'
|
|
summary: Network interface is reporting many receive errors.
|
|
expr: |
|
|
rate(node_network_receive_errs_total{job="node"}[2m]) / rate(node_network_receive_packets_total{job="node"}[2m]) > 0.01
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeNetworkTransmitErrs
|
|
annotations:
|
|
description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
|
|
{{ printf "%.0f" $value }} transmit errors in the last two minutes.'
|
|
summary: Network interface is reporting many transmit errors.
|
|
expr: |
|
|
rate(node_network_transmit_errs_total{job="node"}[2m]) / rate(node_network_transmit_packets_total{job="node"}[2m]) > 0.01
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeHighNumberConntrackEntriesUsed
|
|
annotations:
|
|
description: '{{ $value | humanizePercentage }} of conntrack entries are used.'
|
|
summary: Number of conntrack are getting close to the limit.
|
|
expr: |
|
|
(node_nf_conntrack_entries{job="node"} / node_nf_conntrack_entries_limit) > 0.75
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeTextFileCollectorScrapeError
|
|
annotations:
|
|
description: Node Exporter text file collector on {{ $labels.instance }} failed
|
|
to scrape.
|
|
summary: Node Exporter text file collector failed to scrape.
|
|
expr: |
|
|
node_textfile_scrape_error{job="node"} == 1
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeClockSkewDetected
|
|
annotations:
|
|
description: Clock at {{ $labels.instance }} is out of sync by more than 0.05s.
|
|
Ensure NTP is configured correctly on this host.
|
|
summary: Clock skew detected.
|
|
expr: |
|
|
(
|
|
node_timex_offset_seconds{job="node"} > 0.05
|
|
and
|
|
deriv(node_timex_offset_seconds{job="node"}[5m]) >= 0
|
|
)
|
|
or
|
|
(
|
|
node_timex_offset_seconds{job="node"} < -0.05
|
|
and
|
|
deriv(node_timex_offset_seconds{job="node"}[5m]) <= 0
|
|
)
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeClockNotSynchronising
|
|
annotations:
|
|
description: Clock at {{ $labels.instance }} is not synchronising. Ensure NTP
|
|
is configured on this host.
|
|
summary: Clock not synchronising.
|
|
expr: |
|
|
min_over_time(node_timex_sync_status{job="node"}[5m]) == 0
|
|
and
|
|
node_timex_maxerror_seconds{job="node"} >= 16
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeRAIDDegraded
|
|
annotations:
|
|
description: RAID array '{{ $labels.device }}' at {{ $labels.instance }} is
|
|
in degraded state due to one or more disks failures. Number of spare drives
|
|
is insufficient to fix issue automatically.
|
|
summary: RAID Array is degraded.
|
|
expr: |
|
|
node_md_disks_required{job="node",device!=""} - ignoring (state) (node_md_disks{state="active",job="node",device!=""}) > 0
|
|
for: 15m
|
|
labels:
|
|
severity: critical
|
|
- alert: NodeRAIDDiskFailure
|
|
annotations:
|
|
description: At least one device in RAID array at {{ $labels.instance }} failed.
|
|
Array '{{ $labels.device }}' needs attention and possibly a disk swap.
|
|
summary: Failed device in RAID array.
|
|
expr: |
|
|
node_md_disks{state="failed",job="node",device!=""} > 0
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeFileDescriptorLimit
|
|
annotations:
|
|
description: File descriptors limit at {{ $labels.instance }} is currently at
|
|
{{ printf "%.2f" $value }}%.
|
|
summary: Kernel is predicted to exhaust file descriptors limit soon.
|
|
expr: |
|
|
(
|
|
node_filefd_allocated{job="node"} * 100 / node_filefd_maximum{job="node"} > 70
|
|
)
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeFileDescriptorLimit
|
|
annotations:
|
|
description: File descriptors limit at {{ $labels.instance }} is currently at
|
|
{{ printf "%.2f" $value }}%.
|
|
summary: Kernel is predicted to exhaust file descriptors limit soon.
|
|
expr: |
|
|
(
|
|
node_filefd_allocated{job="node"} * 100 / node_filefd_maximum{job="node"} > 90
|
|
)
|
|
for: 15m
|
|
labels:
|
|
severity: critical
|
|
- alert: NodeCPUHighUsage
|
|
annotations:
|
|
description: |
|
|
CPU usage at {{ $labels.instance }} has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
|
|
summary: High CPU usage.
|
|
expr: |
|
|
sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node", mode!="idle"}[2m]))) * 100 > 90
|
|
for: 15m
|
|
labels:
|
|
severity: info
|
|
- alert: NodeSystemSaturation
|
|
annotations:
|
|
description: |
|
|
System load per core at {{ $labels.instance }} has been above 2 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
|
|
This might indicate this instance resources saturation and can cause it becoming unresponsive.
|
|
summary: System saturated, load per core is very high.
|
|
expr: |
|
|
node_load1{job="node"}
|
|
/ count without (cpu, mode) (node_cpu_seconds_total{job="node", mode="idle"}) > 2
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeMemoryMajorPagesFaults
|
|
annotations:
|
|
description: |
|
|
Memory major pages are occurring at very high rate at {{ $labels.instance }}, 500 major page faults per second for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
|
|
Please check that there is enough memory available at this instance.
|
|
summary: Memory major page faults are occurring at very high rate.
|
|
expr: |
|
|
rate(node_vmstat_pgmajfault{job="node"}[5m]) > 500
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeMemoryHighUtilization
|
|
annotations:
|
|
description: |
|
|
Memory is filling up at {{ $labels.instance }}, has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
|
|
summary: Host is running out of memory.
|
|
expr: |
|
|
100 - (node_memory_MemAvailable_bytes{job="node"} / node_memory_MemTotal_bytes{job="node"} * 100) > 90
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeDiskIOSaturation
|
|
annotations:
|
|
description: |
|
|
Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
|
|
This symptom might indicate disk saturation.
|
|
summary: Disk IO queue is high.
|
|
expr: |
|
|
rate(node_disk_io_time_weighted_seconds_total{job="node", device!=""}[5m]) > 10
|
|
for: 30m
|
|
labels:
|
|
severity: warning
|
|
- alert: NodeSystemdServiceFailed
|
|
annotations:
|
|
description: Systemd service {{ $labels.name }} has entered failed state at
|
|
{{ $labels.instance }}
|
|
summary: Systemd service has entered failed state.
|
|
expr: |
|
|
node_systemd_unit_state{job="node", state="failed"} == 1
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|