This repository has no description
1.9 kB
60 lines
1# vmalert rules. metrics runs against VictoriaMetrics, logs (LogsQL) against
2# VictoriaLogs; vmalert needs one instance per datasource type.
3{ lib }:
4let
5 inherit (import ./default.nix) alwaysOn;
6
7 alert = name: expr: for: summary: {
8 alert = name;
9 inherit expr for;
10 labels.severity = "warning";
11 annotations.summary = summary;
12 };
13in
14{
15 metrics.groups = [
16 {
17 name = "hosts";
18 rules = [
19 # absent_over_time keeps the host label of its selector, so one term
20 # per machine yields one alert per missing machine.
21 (alert "HostDown" (lib.concatMapStringsSep " or " (
22 host: ''absent_over_time(host_uptime{host="${host}"}[10m])''
23 ) alwaysOn) "0m" "{{ $labels.host }} sent no metrics for 10 minutes")
24 # Network mounts (and their autofs placeholders) are reported by
25 # every machine that mounts them; the agent drops tmpfs & co.
26 (alert "DiskAlmostFull"
27 ''host_filesystem_used_ratio{filesystem!~"cifs|smb3|nfs|nfs4|autofs|fuse.sshfs"} > 0.9''
28 "15m"
29 "{{ $labels.host }}: {{ $labels.mountpoint }} is {{ $value | humanizePercentage }} full"
30 )
31 (alert "HighMemory" "1 - host_memory_available_bytes / host_memory_total_bytes > 0.95" "15m"
32 "{{ $labels.host }} uses {{ $value | humanizePercentage }} of its memory"
33 )
34 ];
35 }
36 {
37 name = "gatus";
38 rules = [
39 (alert "GatusEndpointDown" "gatus_results_endpoint_success == 0" "5m"
40 "{{ $labels.name }} fails as seen from {{ $labels.vantage }}"
41 )
42 ];
43 }
44 ];
45
46 logs.groups = [
47 {
48 name = "journal";
49 type = "vlogs";
50 interval = "1m";
51 rules = [
52 (alert "SystemdUnitFailed"
53 ''_time:15m "Failed with result" | stats by (host, unit) count() as failures''
54 "0m"
55 "{{ $labels.unit }} failed on {{ $labels.host }}"
56 )
57 ];
58 }
59 ];
60}