This repository has no description
4.5 kB
160 lines
1# Vector on every machine: host metrics and the journal, written to every
2# monitoring server (dual write). Each sink has its own disk buffer, so a
3# server outage or a laptop being offline doesn't lose data.
4{
5 config,
6 lib,
7 pkgs,
8 ...
9}:
10let
11 inherit (import ./default.nix) servers ports;
12
13 buffer = {
14 type = "disk";
15 max_size = 512 * 1024 * 1024;
16 when_full = "drop_newest";
17 };
18
19 sinksFor = server: {
20 "metrics_${server}" = {
21 type = "prometheus_remote_write";
22 inputs = [
23 "host_metrics"
24 "revision_metric"
25 ];
26 endpoint = "http://${server}.r:${toString ports.victoriametrics}/api/v1/write";
27 healthcheck.enabled = false;
28 inherit buffer;
29 };
30 "logs_${server}" = {
31 type = "elasticsearch";
32 inputs = [ "journal" ];
33 endpoints = [ "http://${server}.r:${toString ports.victorialogs}/insert/elasticsearch/" ];
34 mode = "bulk";
35 api_version = "v8";
36 healthcheck.enabled = false;
37 query = {
38 _msg_field = "message";
39 _time_field = "timestamp";
40 _stream_fields = "host,unit";
41 };
42 inherit buffer;
43 };
44 };
45in
46{
47 services.vector = {
48 enable = true;
49 journaldAccess = true;
50 settings = {
51 data_dir = "/var/lib/vector";
52
53 sources.host_metrics_raw = {
54 type = "host_metrics";
55 collectors = [
56 "cpu"
57 "memory"
58 "disk"
59 "filesystem"
60 "load"
61 "network"
62 "host"
63 ];
64 filesystem.filesystems.excludes = [
65 "tmpfs"
66 "devtmpfs"
67 "overlay"
68 "squashfs"
69 "ramfs"
70 ];
71 scrape_interval_secs = 30;
72 };
73 sources.internal_metrics_raw = {
74 type = "internal_metrics";
75 scrape_interval_secs = 30;
76 };
77 sources.journald_raw.type = "journald";
78
79 # The superconfig revision the machine runs, written by deploy and
80 # autoUpgrade after a switch. Read at runtime: it is in no derivation,
81 # so a new commit doesn't change every machine's closure.
82 sources.revision_raw = {
83 type = "exec";
84 mode = "scheduled";
85 scheduled.exec_interval_secs = 30;
86 command = [
87 pkgs.runtimeShell
88 "-c"
89 "cat /var/lib/superconfig-revision 2>/dev/null || echo unknown"
90 ];
91 include_stderr = false;
92 };
93 transforms.revision = {
94 type = "remap";
95 inputs = [ "revision_raw" ];
96 source = ''
97 rev = strip_whitespace(string!(.message))
98 if rev == "" { rev = "unknown" }
99 . = {
100 "value": 1,
101 "revision": rev,
102 "revision_short": replace(rev, r'^([0-9a-f]{7})[0-9a-f]{33}', "$1"),
103 }
104 '';
105 };
106 transforms.revision_metric = {
107 type = "log_to_metric";
108 inputs = [ "revision" ];
109 metrics = [
110 {
111 type = "gauge";
112 field = "value";
113 name = "nixos_system_info";
114 tags = {
115 host = config.networking.hostName;
116 revision = "{{revision}}";
117 revision_short = "{{revision_short}}";
118 };
119 }
120 ];
121 };
122
123 # The NixOS hostname as host label on every metric, the same value the
124 # journal events get below (alerts and dashboards match on it).
125 transforms.host_metrics = {
126 type = "remap";
127 inputs = [
128 "host_metrics_raw"
129 "internal_metrics_raw"
130 ];
131 source = ''
132 .tags.host = "${config.networking.hostName}"
133 '';
134 };
135
136 # Keep only what we query on: drop debug messages and the many
137 # journald bookkeeping fields. unit prefers UNIT/USER_UNIT, which
138 # systemd sets on its own messages about a unit ("Failed with result"),
139 # so those land in the stream of the unit they are about.
140 transforms.journal = {
141 type = "remap";
142 inputs = [ "journald_raw" ];
143 source = ''
144 if .PRIORITY == "7" { abort }
145 . = {
146 "timestamp": .timestamp,
147 "message": .message,
148 "host": "${config.networking.hostName}",
149 "unit": .UNIT || .USER_UNIT || ._SYSTEMD_USER_UNIT || ._SYSTEMD_UNIT || .SYSLOG_IDENTIFIER || "unknown",
150 "priority": .PRIORITY,
151 "identifier": .SYSLOG_IDENTIFIER,
152 }
153 '';
154 drop_on_abort = true;
155 };
156
157 sinks = lib.mergeAttrsList (map sinksFor servers);
158 };
159 };
160}