mirror of
https://github.com/mdlayher/homelab.git
synced 2024-12-14 11:47:32 +00:00
6f09861ce6
Signed-off-by: Matt Layher <mdlayher@gmail.com>
112 lines
5.1 KiB
Nix
112 lines
5.1 KiB
Nix
{
|
|
groups = [{
|
|
name = "default";
|
|
rules = [
|
|
# PCs which don't run 24/7 are excluded from alerts, and lab-* jobs are
|
|
# excluded due to their experimental nature.
|
|
{
|
|
alert = "InstanceDown";
|
|
expr = ''up{instance!~"(nerr-3.*|theatnerr-1.*)",job!~"lab-.*"} == 0'';
|
|
for = "2m";
|
|
annotations.summary =
|
|
"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.";
|
|
}
|
|
{
|
|
alert = "ServiceDown";
|
|
expr = ''probe_success{instance!~"nerr-3.*",job!~"lab-.*"} == 0'';
|
|
for = "2m";
|
|
annotations.summary =
|
|
"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.";
|
|
}
|
|
{
|
|
alert = "TLSCertificateNearExpiration";
|
|
expr = "probe_ssl_earliest_cert_expiry - time() < 60 * 60 * 24 * 2";
|
|
for = "1m";
|
|
annotations.summary =
|
|
"TLS certificate for {{ $labels.instance }} will expire in less than 2 days.";
|
|
}
|
|
{
|
|
alert = "DiskUsageHigh";
|
|
expr = ''
|
|
(1 - node_filesystem_free_bytes{fstype=~"ext4|vfat"} / node_filesystem_size_bytes) > 0.75'';
|
|
for = "1m";
|
|
annotations.summary =
|
|
"Disk usage on {{ $labels.instance }}:{{ $labels.mountpoint }} ({{ $labels.device }}) exceeds 75%.";
|
|
}
|
|
{
|
|
alert = "APCUPSOnBattery";
|
|
expr = "apcupsd_battery_time_on_seconds > 0";
|
|
annotations.summary =
|
|
"UPS on {{ $labels.instance }} is running on battery power.";
|
|
}
|
|
{
|
|
alert = "APCUPSBatteryTimeLeft";
|
|
expr = "apcupsd_battery_time_left_seconds < 30*60";
|
|
annotations.summary =
|
|
"UPS on {{ $labels.instance }} has less than 30 minutes of remaining battery runtime.";
|
|
}
|
|
# All advertising interfaces should be forwarding IPv6 traffic, and
|
|
# have IPv6 autoconfiguration disabled.
|
|
{
|
|
alert = "CoreRADAdvertisingInterfaceMisconfigured";
|
|
expr = ''
|
|
(corerad_interface_advertising{job="corerad"} == 1) and ((corerad_interface_forwarding == 0) or (corerad_interface_autoconfiguration == 1))'';
|
|
for = "1m";
|
|
annotations.summary =
|
|
"CoreRAD ({{ $labels.instance }}) interface {{ $labels.interface }} is misconfigured for sending IPv6 router advertisements.";
|
|
}
|
|
# All CoreRAD interfaces should multicast IPv6 RAs on a regular basis
|
|
# so hosts don't drop their default route.
|
|
{
|
|
alert = "CoreRADAdvertiserNotMulticasting";
|
|
expr = ''
|
|
rate(corerad_advertiser_router_advertisements_total{job="corerad",type="multicast"}[20m]) == 0'';
|
|
for = "1m";
|
|
annotations.summary =
|
|
"CoreRAD ({{ $labels.instance }}) interface {{ $labels.interface }} has not sent a multicast router advertisment in more than 20 minutes.";
|
|
}
|
|
# Monitor for inconsistent advertisements from hosts on the LAN.
|
|
{
|
|
alert = "CoreRADAdvertiserReceivedInconsistentRouterAdvertisement";
|
|
expr = ''
|
|
rate(corerad_advertiser_router_advertisement_inconsistencies_total{job="corerad"}[5m]) > 0'';
|
|
annotations.summary =
|
|
"CoreRAD ({{ $labels.instance }}) interface {{ $labels.interface }} received an IPv6 router advertisement with inconsistent configuration compared to its own.";
|
|
}
|
|
# We are advertising 2 prefixes per interface out of GUA /56 and ULA /48.
|
|
{
|
|
alert = "CoreRADAdvertiserMissingPrefix";
|
|
expr = ''
|
|
count by (instance, interface) (corerad_advertiser_router_advertisement_prefix_autonomous{job="corerad",prefix=~"2600:6c4a:7880:32.*|fd9e:1a04:f01d:.*"} == 1) != 2'';
|
|
for = "1m";
|
|
annotations.summary =
|
|
"CoreRAD ({{ $labels.instance }}) interface {{ $labels.interface }} is advertising an incorrect number of IPv6 prefixes for SLAAC.";
|
|
}
|
|
# All IPv6 prefixes are advertised with SLAAC.
|
|
{
|
|
alert = "CoreRADAdvertiserPrefixNotAutonomous";
|
|
expr = ''
|
|
corerad_advertiser_router_advertisement_prefix_autonomous{job="corerad"} == 0'';
|
|
for = "1m";
|
|
annotations.summary =
|
|
"CoreRAD ({{ $labels.instance }}) prefix {{ $labels.prefix }} on interface {{ $labels.interface }} is not configured for SLAAC.";
|
|
}
|
|
# Expect regular upstream router advertisements.
|
|
{
|
|
alert = "CoreRADMonitorNoUpstreamRouterAdvertisements";
|
|
expr = ''
|
|
changes(corerad_monitor_messages_received_total{job="corerad",message="router advertisement"}[10m]) == 0'';
|
|
annotations.summary =
|
|
"CoreRAD ({{ $labels.instance }}) interface {{ $labels.interface }} has not received a router advertisement from {{ $labels.host }} in more than 10 minutes.";
|
|
}
|
|
# Expect continuous upstream router advertisements.
|
|
{
|
|
alert = "CoreRADMonitorDefaultRouteExpiring";
|
|
expr = ''
|
|
corerad_monitor_default_route_expiration_time{job="corerad"} - time() < 2*60*60'';
|
|
annotations.summary =
|
|
"CoreRAD ({{ $labels.instance }}) interface {{ $labels.interface }} will drop its default route to {{ $labels.router }} in less than 2 hours.";
|
|
}
|
|
];
|
|
}];
|
|
}
|