This repository has no description
configs.monitoring: alert on disk health and memory pressure
smartctl_exporter runs on every non-container machine, bound to
localhost and scraped by Vector into the host-tagged metrics path.
Alerts on a failed SMART status, NVMe critical warnings, new NVMe media
errors and NVMe spare below its threshold.
Memory PSI comes from /proc/pressure/memory via an exec source, as a
counter of stalled seconds, so the alert takes an exact 10m rate instead
of the kernel's 5m avg300 EWMA. Plus an alert on any OOM kill.
MemoryPressure threshold 10%: some task stalling on memory a tenth of
the time over 10 minutes is sustained reclaim/swap thrash, while short
spikes such as big builds average out.
author
Hermes
committer
lassulus
date
(Oct 16, 2026, 11:03 AM CEST)
commit
fe1f627a
fe1f627a751719bd4e2a05d8e33b83f882ce2f2f
parent
8e69555c
8e69555ce428ae2e9c3e96a214f3dfcbf0f03d4b
+75
-1
+56
-1
configs/monitoring/agent.nix
+56
-1
configs/monitoring/agent.nix
···
10
10
let
11
11
inherit (import ./default.nix) servers ports;
12
12
13
+
# Containers have no disks of their own and lack the capabilities the
14
+
# SMART exporter needs.
15
+
smart = !config.boot.isContainer;
16
+
13
17
buffer = {
14
18
type = "disk";
15
19
max_size = 512 * 1024 * 1024;
···
22
26
inputs = [
23
27
"host_metrics"
24
28
"revision_metric"
29
+
"memory_pressure_metric"
25
30
];
26
31
endpoint = "http://${server}.r:${toString ports.victoriametrics}/api/v1/write";
27
32
healthcheck.enabled = false;
···
44
49
};
45
50
in
46
51
{
52
+
services.prometheus.exporters.smartctl = lib.mkIf smart {
53
+
enable = true;
54
+
listenAddress = "127.0.0.1";
55
+
};
56
+
47
57
services.vector = {
48
58
enable = true;
49
59
journaldAccess = true;
···
75
85
scrape_interval_secs = 30;
76
86
};
77
87
sources.journald_raw.type = "journald";
88
+
sources.smartctl_raw = lib.mkIf smart {
89
+
type = "prometheus_scrape";
90
+
endpoints = [
91
+
"http://127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}/metrics"
92
+
];
93
+
scrape_interval_secs = 60;
94
+
};
95
+
96
+
# Memory PSI: the time some task stalled on memory, as a counter in
97
+
# seconds, so rate() over it is the stalled fraction of a window.
98
+
sources.memory_pressure_raw = {
99
+
type = "exec";
100
+
mode = "scheduled";
101
+
scheduled.exec_interval_secs = 30;
102
+
command = [
103
+
"${pkgs.coreutils}/bin/cat"
104
+
"/proc/pressure/memory"
105
+
];
106
+
include_stderr = false;
107
+
};
108
+
transforms.memory_pressure = {
109
+
type = "remap";
110
+
inputs = [ "memory_pressure_raw" ];
111
+
source = ''
112
+
fields = parse_key_value!(.message)
113
+
if !exists(fields.some) { abort }
114
+
. = { "seconds": to_int!(fields.total) / 1000000 }
115
+
'';
116
+
drop_on_abort = true;
117
+
};
118
+
transforms.memory_pressure_metric = {
119
+
type = "log_to_metric";
120
+
inputs = [ "memory_pressure" ];
121
+
metrics = [
122
+
{
123
+
type = "counter";
124
+
kind = "absolute";
125
+
increment_by_value = true;
126
+
field = "seconds";
127
+
name = "host_pressure_memory_some_seconds_total";
128
+
tags.host = config.networking.hostName;
129
+
}
130
+
];
131
+
};
78
132
79
133
# The superconfig revision the machine runs, written by deploy and
80
134
# autoUpgrade after a switch. Read at runtime: it is in no derivation,
···
127
181
inputs = [
128
182
"host_metrics_raw"
129
183
"internal_metrics_raw"
130
-
];
184
+
]
185
+
++ lib.optional smart "smartctl_raw";
131
186
source = ''
132
187
.tags.host = "${config.networking.hostName}"
133
188
'';
+19
configs/monitoring/alerts.nix
+19
configs/monitoring/alerts.nix
···
46
46
(alert "HighMemory" "1 - host_memory_available_bytes / host_memory_total_bytes > 0.95" "15m"
47
47
"{{ $labels.host }} uses {{ $value | humanizePercentage }} of its memory"
48
48
)
49
+
(alert "MemoryPressure" "rate(host_pressure_memory_some_seconds_total[10m]) > 0.1" "5m"
50
+
"{{ $labels.host }}: tasks stalled on memory {{ $value | humanizePercentage }} of the time"
51
+
)
52
+
(alert "OOMKill" "increase(host_memory_oom_kill_events_total[10m]) > 0" "0m"
53
+
"{{ $labels.host }}: {{ $value }} processes OOM-killed in the last 10 minutes"
54
+
)
55
+
(alert "SmartFailed" "smartctl_device_smart_status == 0" "5m"
56
+
"{{ $labels.host }}: {{ $labels.device }} fails its SMART health check"
57
+
)
58
+
(alert "NvmeCriticalWarning" "smartctl_device_critical_warning != 0" "5m"
59
+
"{{ $labels.host }}: {{ $labels.device }} reports critical warning {{ $value }}"
60
+
)
61
+
(alert "NvmeMediaErrors" "increase(smartctl_device_media_errors[1h]) > 0" "0m"
62
+
"{{ $labels.host }}: {{ $labels.device }} had {{ $value }} new media errors in the last hour"
63
+
)
64
+
(alert "NvmeSpareLow" "smartctl_device_available_spare < smartctl_device_available_spare_threshold"
65
+
"5m"
66
+
"{{ $labels.host }}: {{ $labels.device }} has only {{ $value }}% spare left"
67
+
)
49
68
];
50
69
}
51
70
{