This repository has no description
9.3 kB
208 lines
1# inspiration from https://github.com/Mic92/dotfiles/blob/master/nixos/eva/modules/prometheus/alert-rules.nix
2{ lib }:
3
4lib.mapAttrsToList
5 (name: opts: {
6 alert = name;
7 expr = opts.condition;
8 for = opts.time or "2m";
9 labels = { };
10 annotations.description = opts.description;
11 })
12 ({
13 prometheus_too_many_restarts = {
14 condition = ''changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager|telegraf"}[15m]) > 2'';
15 description = "Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.";
16 };
17
18 alert_manager_config_not_synced = {
19 condition = ''count(count_values("config_hash", alertmanager_config_hash)) > 1'';
20 description = "Configurations of AlertManager cluster instances are out of sync.";
21 };
22
23 prometheus_not_connected_to_alertmanager = {
24 condition = "prometheus_notifications_alertmanagers_discovered < 1";
25 description = "Prometheus cannot connect the alertmanager\n VALUE = {{ $value }}\n LABELS = {{ $labels }}";
26 };
27
28 prometheus_rule_evaluation_failures = {
29 condition = "increase(prometheus_rule_evaluation_failures_total[3m]) > 0";
30 description = "Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}";
31 };
32
33 prometheus_template_expansion_failures = {
34 condition = "increase(prometheus_template_text_expansion_failures_total[3m]) > 0";
35 time = "0m";
36 description = "Prometheus encountered {{ $value }} template text expansion failures\n VALUE = {{ $value }}\n LABELS = {{ $labels }}";
37 };
38
39 promtail_request_errors = {
40 condition = ''100 * sum(rate(promtail_request_duration_seconds_count{status_code=~"5..|failed"}[1m])) by (namespace, job, route, instance) / sum(rate(promtail_request_duration_seconds_count[1m])) by (namespace, job, route, instance) > 10'';
41 time = "15m";
42 description = ''{{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors.'';
43 };
44
45 promtail_file_lagging = {
46 condition = ''abs(promtail_file_bytes_total - promtail_read_bytes_total) > 1e6'';
47 time = "15m";
48 description = ''{{ $labels.instance }} {{ $labels.job }} {{ $labels.path }} has been lagging by more than 1MB for more than 15m.'';
49 };
50
51 filesystem_full_80percent = {
52 condition = ''disk_used_percent{mode!="ro"} >= 95'';
53 time = "10m";
54 description = "{{$labels.instance}} device {{$labels.device}} on {{$labels.path}} got less than 20% space left on its filesystem.";
55 };
56
57 filesystem_full_krebs = {
58 condition = ''disk_used_percent{mode!="ro", org="krebs"} >= 95'';
59 time = "10m";
60 description = "{{$labels.instance}} device {{$labels.device}} on {{$labels.path}} got less than 5% space left on its filesystem.";
61 };
62
63 filesystem_inodes_full = {
64 condition = ''disk_inodes_free / disk_inodes_total < 0.10'';
65 time = "10m";
66 description = "{{$labels.instance}} device {{$labels.device}} on {{$labels.path}} got less than 10% inodes left on its filesystem.";
67 };
68
69 daily_task_not_run = {
70 # give 6 hours grace period
71 condition = ''time() - task_last_run{state="ok",frequency="daily"} > (24 + 6) * 60 * 60'';
72 description = "{{$labels.host}}: {{$labels.name}} was not run in the last 24h";
73 };
74
75 daily_task_failed = {
76 condition = ''task_last_run{state="fail"}'';
77 description = "{{$labels.host}}: {{$labels.name}} failed to run";
78 };
79
80 swap_using_30percent = {
81 condition = "mem_swap_total - (mem_swap_cached + mem_swap_free) > mem_swap_total * 0.3";
82 time = "30m";
83 description = "{{$labels.host}} is using 30% of its swap space for at least 30 minutes.";
84 };
85
86 systemd_service_failed = {
87 condition = ''systemd_units_active_code{name!~"nixpkgs-update-.*.service"} == 3'';
88 description = "{{$labels.host}} failed to (re)start service {{$labels.name}}.";
89 };
90
91 service_not_running = {
92 condition = ''systemd_units_active_code{name=~"teamspeak3-server.service|tt-rss.service", sub!="running"}'';
93 description = "{{$labels.host}} should have a running {{$labels.name}}.";
94 };
95
96 nfs_export_not_present = {
97 condition = "nfs_export_present == 0";
98 time = "1h";
99 description = "{{$labels.host}} cannot reach nfs export [{{$labels.server}}]:{{$labels.path}}";
100 };
101
102 ram_using_90percent = {
103 condition = "mem_buffered + mem_free + mem_cached < mem_total * 0.1";
104 time = "1h";
105 description = "{{$labels.host}} is using at least 90% of its RAM for at least 1 hour.";
106 };
107 load15 = {
108 condition = ''system_load15 / system_n_cpus{org!="nix-community"} >= 2.0'';
109 time = "10m";
110 description = "{{$labels.host}} is running with load15 > 1 for at least 5 minutes: {{$value}}";
111 };
112 reboot = {
113 condition = "system_uptime < 300";
114 description = "{{$labels.host}} just rebooted.";
115 };
116 uptime = {
117 # too scared to upgrade matchbox
118 condition = ''system_uptime {host!~"^(matchbox|grandalf)$"} > 2592000'';
119 description = "Uptime monster: {{$labels.host}} has been up for more than 30 days.";
120 };
121 telegraf_down = {
122 condition = ''min(up{job=~"telegraf",type!='mobile'}) by (source, job, instance, org) == 0'';
123 time = "3m";
124 description = "{{$labels.instance}}: {{$labels.job}} telegraf exporter from {{$labels.source}} is down.";
125 };
126 ping = {
127 condition = "ping_result_code{type!='mobile'} != 0";
128 description = "{{$labels.url}}: ping from {{$labels.instance}} has failed!";
129 };
130 ping_high_latency = {
131 condition = "ping_average_response_ms{type!='mobile'} > 5000";
132 description = "{{$labels.instance}}: ping probe from {{$labels.source}} is encountering high latency!";
133 };
134 http = {
135 condition = "http_response_result_code != 0";
136 description = "{{$labels.server}} : http request failed from {{$labels.instance}}: {{$labels.result}}!";
137 };
138 http_match_failed = {
139 condition = "http_response_response_string_match == 0";
140 description = "{{$labels.server}} : http body not as expected; status code: {{$labels.status_code}}!";
141 };
142 dns_query = {
143 condition = "dns_query_result_code != 0";
144 description = "{{$labels.domain}} : could retrieve A record {{$labels.instance}} from server {{$labels.server}}: {{$labels.result}}!";
145 };
146 secure_dns_query = {
147 condition = "secure_dns_state != 0";
148 description = "{{$labels.domain}} : could retrieve A record {{$labels.instance}} from server {{$labels.server}}: {{$labels.result}} for protocol {{$labels.protocol}}!";
149 };
150 connection_failed = {
151 condition = "net_response_result_code != 0";
152 description = "{{$labels.server}}: connection to {{$labels.port}}({{$labels.protocol}}) failed from {{$labels.instance}}";
153 };
154 healthchecks = {
155 condition = "hc_check_up == 0";
156 description = "{{$labels.instance}}: healtcheck {{$labels.job}} fails!";
157 };
158 cert_expiry = {
159 condition = "x509_cert_expiry < 7*24*3600";
160 description = "{{$labels.instance}}: The TLS certificate from {{$labels.source}} will expire in less than 7 days: {{$value}}s";
161 };
162
163 postfix_queue_length = {
164 condition = "avg_over_time(postfix_queue_length[1h]) > 10";
165 description = "{{$labels.instance}}: postfix mail queue has undelivered {{$value}} items";
166 };
167
168 zfs_errors = {
169 condition = "zfs_arcstats_l2_io_error + zfs_dmu_tx_error + zfs_arcstats_l2_writes_error > 0";
170 description = "{{$labels.instance}} reports: {{$value}} ZFS IO errors.";
171 };
172
173 # ignore devices that disabled S.M.A.R.T (example if attached via USB)
174 smart_errors = {
175 condition = ''smart_device_health_ok{enabled!="Disabled"} != 1'';
176 description = "{{$labels.instance}}: S.M.A.R.T reports: {{$labels.device}} ({{$labels.model}}) has errors.";
177 };
178
179 oom_kills = {
180 condition = "increase(kernel_vmstat_oom_kill[5m]) > 0";
181 description = "{{$labels.instance}}: OOM kill detected";
182 };
183
184 unusual_disk_read_latency = {
185 condition = "rate(diskio_read_time[1m]) / rate(diskio_reads[1m]) > 0.1 and rate(diskio_reads[1m]) > 0";
186 description = "{{$labels.instance}}: Disk latency is growing (read operations > 100ms)\n";
187 };
188
189 unusual_disk_write_latency = {
190 condition = "rate(diskio_write_time[1m]) / rate(diskio_write[1m]) > 0.1 and rate(diskio_write[1m]) > 0";
191 description = "{{$labels.instance}}: Disk latency is growing (write operations > 100ms)\n";
192 };
193
194 host_memory_under_memory_pressure = {
195 condition = "rate(node_vmstat_pgmajfault[1m]) > 1000";
196 description = "{{$labels.instance}}: The node is under heavy memory pressure. High rate of major page faults: {{$value}}";
197 };
198
199 ext4_errors = {
200 condition = "ext4_errors_value > 0";
201 description = "{{$labels.instance}}: ext4 has reported {{$value}} I/O errors: check /sys/fs/ext4/*/errors_count";
202 };
203
204 alerts_silences_changed = {
205 condition = ''abs(delta(alertmanager_silences{state="active"}[1h])) >= 1'';
206 description = "alertmanager: number of active silences has changed: {{$value}}";
207 };
208 })