This repository has no description
6.7 kB
225 lines
1# Vector on every machine: host metrics and the journal, written to every
2# monitoring server (dual write). Each sink has its own disk buffer, so a
3# server outage or a laptop being offline doesn't lose data.
4{
5 config,
6 lib,
7 pkgs,
8 ...
9}:
10let
11 inherit (import ./default.nix) servers ports;
12
13 # Containers have no disks of their own and lack the capabilities the
14 # SMART exporter needs.
15 smart = !config.boot.isContainer;
16
17 buffer = {
18 type = "disk";
19 max_size = 512 * 1024 * 1024;
20 when_full = "drop_newest";
21 };
22
23 sinksFor = server: {
24 "metrics_${server}" = {
25 type = "prometheus_remote_write";
26 inputs = [
27 "host_metrics"
28 "revision_metric"
29 "memory_pressure_metric"
30 ]
31 ++ config.monitoring.extraMetrics;
32 endpoint = "http://${server}.r:${toString ports.victoriametrics}/api/v1/write";
33 healthcheck.enabled = false;
34 inherit buffer;
35 };
36 "logs_${server}" = {
37 type = "elasticsearch";
38 inputs = [ "journal" ];
39 endpoints = [ "http://${server}.r:${toString ports.victorialogs}/insert/elasticsearch/" ];
40 mode = "bulk";
41 api_version = "v8";
42 healthcheck.enabled = false;
43 query = {
44 _msg_field = "message";
45 _time_field = "timestamp";
46 _stream_fields = "host,unit";
47 };
48 inherit buffer;
49 };
50 };
51in
52{
53 # Vector components of machine specific metrics, defined by other modules
54 # (e.g. configs/gg23/internet.nix). They have to set the host tag
55 # themselves, like transforms.host_metrics does.
56 options.monitoring.extraMetrics = lib.mkOption {
57 type = lib.types.listOf lib.types.str;
58 default = [ ];
59 description = "Vector component ids added to the inputs of the metric sinks.";
60 };
61
62 config.services.prometheus.exporters.smartctl = lib.mkIf smart {
63 enable = true;
64 listenAddress = "127.0.0.1";
65 };
66
67 config.services.vector = {
68 enable = true;
69 journaldAccess = true;
70 settings = {
71 data_dir = "/var/lib/vector";
72
73 sources.host_metrics_raw = {
74 type = "host_metrics";
75 collectors = [
76 "cpu"
77 "memory"
78 "disk"
79 "filesystem"
80 "load"
81 "network"
82 "host"
83 ];
84 filesystem.filesystems.excludes = [
85 "tmpfs"
86 "devtmpfs"
87 "overlay"
88 "squashfs"
89 "ramfs"
90 ];
91 scrape_interval_secs = 30;
92 };
93 sources.internal_metrics_raw = {
94 type = "internal_metrics";
95 scrape_interval_secs = 30;
96 };
97 sources.journald_raw.type = "journald";
98 sources.smartctl_raw = lib.mkIf smart {
99 type = "prometheus_scrape";
100 endpoints = [
101 "http://127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}/metrics"
102 ];
103 scrape_interval_secs = 60;
104 };
105
106 # Memory PSI: the time some task stalled on memory, as a counter in
107 # seconds, so rate() over it is the stalled fraction of a window.
108 sources.memory_pressure_raw = {
109 type = "exec";
110 mode = "scheduled";
111 scheduled.exec_interval_secs = 30;
112 command = [
113 "${pkgs.coreutils}/bin/cat"
114 "/proc/pressure/memory"
115 ];
116 include_stderr = false;
117 };
118 transforms.memory_pressure = {
119 type = "remap";
120 inputs = [ "memory_pressure_raw" ];
121 source = ''
122 fields = parse_key_value!(.message)
123 if !exists(fields.some) { abort }
124 . = { "seconds": to_int!(fields.total) / 1000000 }
125 '';
126 drop_on_abort = true;
127 };
128 transforms.memory_pressure_metric = {
129 type = "log_to_metric";
130 inputs = [ "memory_pressure" ];
131 metrics = [
132 {
133 type = "counter";
134 kind = "absolute";
135 increment_by_value = true;
136 field = "seconds";
137 name = "host_pressure_memory_some_seconds_total";
138 tags.host = config.networking.hostName;
139 }
140 ];
141 };
142
143 # The superconfig revision the machine runs, written by deploy and
144 # autoUpgrade after a switch. Read at runtime: it is in no derivation,
145 # so a new commit doesn't change every machine's closure.
146 sources.revision_raw = {
147 type = "exec";
148 mode = "scheduled";
149 scheduled.exec_interval_secs = 30;
150 command = [
151 pkgs.runtimeShell
152 "-c"
153 "cat /var/lib/superconfig-revision 2>/dev/null || echo unknown"
154 ];
155 include_stderr = false;
156 };
157 transforms.revision = {
158 type = "remap";
159 inputs = [ "revision_raw" ];
160 source = ''
161 rev = strip_whitespace(string!(.message))
162 if rev == "" { rev = "unknown" }
163 . = {
164 "value": 1,
165 "revision": rev,
166 "revision_short": replace(rev, r'^([0-9a-f]{7})[0-9a-f]{33}', "$1"),
167 }
168 '';
169 };
170 transforms.revision_metric = {
171 type = "log_to_metric";
172 inputs = [ "revision" ];
173 metrics = [
174 {
175 type = "gauge";
176 field = "value";
177 name = "nixos_system_info";
178 tags = {
179 host = config.networking.hostName;
180 revision = "{{revision}}";
181 revision_short = "{{revision_short}}";
182 };
183 }
184 ];
185 };
186
187 # The NixOS hostname as host label on every metric, the same value the
188 # journal events get below (alerts and dashboards match on it).
189 transforms.host_metrics = {
190 type = "remap";
191 inputs = [
192 "host_metrics_raw"
193 "internal_metrics_raw"
194 ]
195 ++ lib.optional smart "smartctl_raw";
196 source = ''
197 .tags.host = "${config.networking.hostName}"
198 '';
199 };
200
201 # Keep only what we query on: drop debug messages and the many
202 # journald bookkeeping fields. unit prefers UNIT/USER_UNIT, which
203 # systemd sets on its own messages about a unit ("Failed with result"),
204 # so those land in the stream of the unit they are about.
205 transforms.journal = {
206 type = "remap";
207 inputs = [ "journald_raw" ];
208 source = ''
209 if .PRIORITY == "7" { abort }
210 . = {
211 "timestamp": .timestamp,
212 "message": .message,
213 "host": "${config.networking.hostName}",
214 "unit": .UNIT || .USER_UNIT || ._SYSTEMD_USER_UNIT || ._SYSTEMD_UNIT || .SYSLOG_IDENTIFIER || "unknown",
215 "priority": .PRIORITY,
216 "identifier": .SYSLOG_IDENTIFIER,
217 }
218 '';
219 drop_on_abort = true;
220 };
221
222 sinks = lib.mergeAttrsList (map sinksFor servers);
223 };
224 };
225}