This repository has no description
3.1 kB
108 lines
1# Vector on every machine: host metrics and the journal, written to every
2# monitoring server (dual write). Each sink has its own disk buffer, so a
3# server outage or a laptop being offline doesn't lose data.
4{ config, lib, ... }:
5let
6 inherit (import ./default.nix) servers ports;
7
8 buffer = {
9 type = "disk";
10 max_size = 512 * 1024 * 1024;
11 when_full = "drop_newest";
12 };
13
14 sinksFor = server: {
15 "metrics_${server}" = {
16 type = "prometheus_remote_write";
17 inputs = [ "host_metrics" ];
18 endpoint = "http://${server}.r:${toString ports.victoriametrics}/api/v1/write";
19 healthcheck.enabled = false;
20 inherit buffer;
21 };
22 "logs_${server}" = {
23 type = "elasticsearch";
24 inputs = [ "journal" ];
25 endpoints = [ "http://${server}.r:${toString ports.victorialogs}/insert/elasticsearch/" ];
26 mode = "bulk";
27 api_version = "v8";
28 healthcheck.enabled = false;
29 query = {
30 _msg_field = "message";
31 _time_field = "timestamp";
32 _stream_fields = "host,unit";
33 };
34 inherit buffer;
35 };
36 };
37in
38{
39 services.vector = {
40 enable = true;
41 journaldAccess = true;
42 settings = {
43 data_dir = "/var/lib/vector";
44
45 sources.host_metrics_raw = {
46 type = "host_metrics";
47 collectors = [
48 "cpu"
49 "memory"
50 "disk"
51 "filesystem"
52 "load"
53 "network"
54 "host"
55 ];
56 filesystem.filesystems.excludes = [
57 "tmpfs"
58 "devtmpfs"
59 "overlay"
60 "squashfs"
61 "ramfs"
62 ];
63 scrape_interval_secs = 30;
64 };
65 sources.internal_metrics_raw = {
66 type = "internal_metrics";
67 scrape_interval_secs = 30;
68 };
69 sources.journald_raw.type = "journald";
70
71 # The NixOS hostname as host label on every metric, the same value the
72 # journal events get below (alerts and dashboards match on it).
73 transforms.host_metrics = {
74 type = "remap";
75 inputs = [
76 "host_metrics_raw"
77 "internal_metrics_raw"
78 ];
79 source = ''
80 .tags.host = "${config.networking.hostName}"
81 '';
82 };
83
84 # Keep only what we query on: drop debug messages and the many
85 # journald bookkeeping fields. unit prefers UNIT/USER_UNIT, which
86 # systemd sets on its own messages about a unit ("Failed with result"),
87 # so those land in the stream of the unit they are about.
88 transforms.journal = {
89 type = "remap";
90 inputs = [ "journald_raw" ];
91 source = ''
92 if .PRIORITY == "7" { abort }
93 . = {
94 "timestamp": .timestamp,
95 "message": .message,
96 "host": "${config.networking.hostName}",
97 "unit": .UNIT || .USER_UNIT || ._SYSTEMD_USER_UNIT || ._SYSTEMD_UNIT || .SYSLOG_IDENTIFIER || "unknown",
98 "priority": .PRIORITY,
99 "identifier": .SYSLOG_IDENTIFIER,
100 }
101 '';
102 drop_on_abort = true;
103 };
104
105 sinks = lib.mergeAttrsList (map sinksFor servers);
106 };
107 };
108}