This repository has no description
2.7 kB
75 lines
1# vmalert rules. metrics runs against VictoriaMetrics, logs (LogsQL) against
2# VictoriaLogs; vmalert needs one instance per datasource type.
3{ lib }:
4let
5 inherit (import ./default.nix) alwaysOn;
6
7 alert = name: expr: for: summary: {
8 alert = name;
9 inherit expr for;
10 labels.severity = "warning";
11 annotations.summary = summary;
12 };
13in
14{
15 metrics.groups = [
16 {
17 name = "hosts";
18 rules = [
19 # absent_over_time keeps the host label of its selector, so one term
20 # per machine yields one alert per missing machine.
21 (alert "HostDown" (lib.concatMapStringsSep " or " (
22 host: ''absent_over_time(host_uptime{host="${host}"}[10m])''
23 ) alwaysOn) "0m" "{{ $labels.host }} sent no metrics for 10 minutes")
24 # Network mounts (and their autofs placeholders) are reported by
25 # every machine that mounts them; the agent drops tmpfs & co.
26 # neoprism's /var/download is always above 90%, see DownloadSpace*.
27 (alert "DiskAlmostFull"
28 ''host_filesystem_used_ratio{filesystem!~"cifs|smb3|nfs|nfs4|autofs|fuse.sshfs"} > 0.9 unless on (host, mountpoint) host_filesystem_used_ratio{host="neoprism",mountpoint="/var/download"}''
29 "15m"
30 "{{ $labels.host }}: {{ $labels.mountpoint }} is {{ $value | humanizePercentage }} full"
31 )
32 (alert "DownloadSpaceLow"
33 ''host_filesystem_free_bytes{host="neoprism",mountpoint="/var/download"} < 200e9''
34 "15m"
35 "{{ $labels.host }}: {{ $labels.mountpoint }} has only {{ $value | humanize }}B free"
36 )
37 (
38 alert "DownloadSpaceCritical"
39 ''host_filesystem_free_bytes{host="neoprism",mountpoint="/var/download"} < 100e9''
40 "15m"
41 "{{ $labels.host }}: {{ $labels.mountpoint }} has only {{ $value | humanize }}B free"
42 // {
43 labels.severity = "critical";
44 }
45 )
46 (alert "HighMemory" "1 - host_memory_available_bytes / host_memory_total_bytes > 0.95" "15m"
47 "{{ $labels.host }} uses {{ $value | humanizePercentage }} of its memory"
48 )
49 ];
50 }
51 {
52 name = "gatus";
53 rules = [
54 (alert "GatusEndpointDown" "gatus_results_endpoint_success == 0" "5m"
55 "{{ $labels.name }} fails as seen from {{ $labels.vantage }}"
56 )
57 ];
58 }
59 ];
60
61 logs.groups = [
62 {
63 name = "journal";
64 type = "vlogs";
65 interval = "1m";
66 rules = [
67 (alert "SystemdUnitFailed"
68 ''_time:15m "Failed with result" | stats by (host, unit) count() as failures''
69 "0m"
70 "{{ $labels.unit }} failed on {{ $labels.host }}"
71 )
72 ];
73 }
74 ];
75}