This repository has no description
configs.monitoring: vector agents, victoria* + alertmanager stack on styx/coaxmetal
Every NixOS machine runs Vector (host metrics, internal metrics, journal)
and writes to VictoriaMetrics/VictoriaLogs on both styx and coaxmetal,
each sink with a 512 MiB disk buffer. Both servers run vmalert (metrics and
LogsQL instances), a two-node Alertmanager cluster posting to Matrix via
matrix-alertmanager-receiver, Gatus (results scraped into VictoriaMetrics,
alerted on by vmalert) and Grafana with provisioned datasources and
dashboards. All ports are open on retiolum only.
Replaces telegraf on neoprism, including the borgbackup status file.
author
Hermes
committer
lassulus
date
(Oct 14, 2026, 12:28 PM CEST)
commit
e731cb83
e731cb835ff8310562d9a679739528c550a24458
parent
47d593b9
47d593b9953b2ba46443e09d49eef5306da8ed07
+748
-205
+1
configs/default.nix
+1
configs/default.nix
+108
configs/monitoring/agent.nix
+108
configs/monitoring/agent.nix
···
1
+
# Vector on every machine: host metrics and the journal, written to every
2
+
# monitoring server (dual write). Each sink has its own disk buffer, so a
3
+
# server outage or a laptop being offline doesn't lose data.
4
+
{ config, lib, ... }:
5
+
let
6
+
inherit (import ./default.nix) servers ports;
7
+
8
+
buffer = {
9
+
type = "disk";
10
+
max_size = 512 * 1024 * 1024;
11
+
when_full = "drop_newest";
12
+
};
13
+
14
+
sinksFor = server: {
15
+
"metrics_${server}" = {
16
+
type = "prometheus_remote_write";
17
+
inputs = [ "host_metrics" ];
18
+
endpoint = "http://${server}.r:${toString ports.victoriametrics}/api/v1/write";
19
+
healthcheck.enabled = false;
20
+
inherit buffer;
21
+
};
22
+
"logs_${server}" = {
23
+
type = "elasticsearch";
24
+
inputs = [ "journal" ];
25
+
endpoints = [ "http://${server}.r:${toString ports.victorialogs}/insert/elasticsearch/" ];
26
+
mode = "bulk";
27
+
api_version = "v8";
28
+
healthcheck.enabled = false;
29
+
query = {
30
+
_msg_field = "message";
31
+
_time_field = "timestamp";
32
+
_stream_fields = "host,unit";
33
+
};
34
+
inherit buffer;
35
+
};
36
+
};
37
+
in
38
+
{
39
+
services.vector = {
40
+
enable = true;
41
+
journaldAccess = true;
42
+
settings = {
43
+
data_dir = "/var/lib/vector";
44
+
45
+
sources.host_metrics_raw = {
46
+
type = "host_metrics";
47
+
collectors = [
48
+
"cpu"
49
+
"memory"
50
+
"disk"
51
+
"filesystem"
52
+
"load"
53
+
"network"
54
+
"host"
55
+
];
56
+
filesystem.filesystems.excludes = [
57
+
"tmpfs"
58
+
"devtmpfs"
59
+
"overlay"
60
+
"squashfs"
61
+
"ramfs"
62
+
];
63
+
scrape_interval_secs = 30;
64
+
};
65
+
sources.internal_metrics_raw = {
66
+
type = "internal_metrics";
67
+
scrape_interval_secs = 30;
68
+
};
69
+
sources.journald_raw.type = "journald";
70
+
71
+
# The NixOS hostname as host label on every metric, the same value the
72
+
# journal events get below (alerts and dashboards match on it).
73
+
transforms.host_metrics = {
74
+
type = "remap";
75
+
inputs = [
76
+
"host_metrics_raw"
77
+
"internal_metrics_raw"
78
+
];
79
+
source = ''
80
+
.tags.host = "${config.networking.hostName}"
81
+
'';
82
+
};
83
+
84
+
# Keep only what we query on: drop debug messages and the many
85
+
# journald bookkeeping fields. unit prefers UNIT/USER_UNIT, which
86
+
# systemd sets on its own messages about a unit ("Failed with result"),
87
+
# so those land in the stream of the unit they are about.
88
+
transforms.journal = {
89
+
type = "remap";
90
+
inputs = [ "journald_raw" ];
91
+
source = ''
92
+
if .PRIORITY == "7" { abort }
93
+
. = {
94
+
"timestamp": .timestamp,
95
+
"message": .message,
96
+
"host": "${config.networking.hostName}",
97
+
"unit": .UNIT || .USER_UNIT || ._SYSTEMD_USER_UNIT || ._SYSTEMD_UNIT || .SYSLOG_IDENTIFIER || "unknown",
98
+
"priority": .PRIORITY,
99
+
"identifier": .SYSLOG_IDENTIFIER,
100
+
}
101
+
'';
102
+
drop_on_abort = true;
103
+
};
104
+
105
+
sinks = lib.mergeAttrsList (map sinksFor servers);
106
+
};
107
+
};
108
+
}
+56
configs/monitoring/alerts.nix
+56
configs/monitoring/alerts.nix
···
1
+
# vmalert rules. metrics runs against VictoriaMetrics, logs (LogsQL) against
2
+
# VictoriaLogs; vmalert needs one instance per datasource type.
3
+
{ lib }:
4
+
let
5
+
inherit (import ./default.nix) alwaysOn;
6
+
7
+
alert = name: expr: for: summary: {
8
+
alert = name;
9
+
inherit expr for;
10
+
labels.severity = "warning";
11
+
annotations.summary = summary;
12
+
};
13
+
in
14
+
{
15
+
metrics.groups = [
16
+
{
17
+
name = "hosts";
18
+
rules = [
19
+
# absent_over_time keeps the host label of its selector, so one term
20
+
# per machine yields one alert per missing machine.
21
+
(alert "HostDown" (lib.concatMapStringsSep " or " (
22
+
host: ''absent_over_time(host_uptime{host="${host}"}[10m])''
23
+
) alwaysOn) "0m" "{{ $labels.host }} sent no metrics for 10 minutes")
24
+
(alert "DiskAlmostFull" "host_filesystem_used_ratio > 0.9" "15m"
25
+
"{{ $labels.host }}: {{ $labels.mountpoint }} is {{ $value | humanizePercentage }} full"
26
+
)
27
+
(alert "HighMemory" "1 - host_memory_available_bytes / host_memory_total_bytes > 0.95" "15m"
28
+
"{{ $labels.host }} uses {{ $value | humanizePercentage }} of its memory"
29
+
)
30
+
];
31
+
}
32
+
{
33
+
name = "gatus";
34
+
rules = [
35
+
(alert "GatusEndpointDown" "gatus_results_endpoint_success == 0" "5m"
36
+
"{{ $labels.name }} fails as seen from {{ $labels.vantage }}"
37
+
)
38
+
];
39
+
}
40
+
];
41
+
42
+
logs.groups = [
43
+
{
44
+
name = "journal";
45
+
type = "vlogs";
46
+
interval = "1m";
47
+
rules = [
48
+
(alert "SystemdUnitFailed"
49
+
''_time:15m "Failed with result" | stats by (host, unit) count() as failures''
50
+
"0m"
51
+
"{{ $labels.unit }} failed on {{ $labels.host }}"
52
+
)
53
+
];
54
+
}
55
+
];
56
+
}
+73
configs/monitoring/checks.nix
+73
configs/monitoring/checks.nix
···
1
+
# Gatus endpoints, the same on every monitoring server. Gatus itself doesn't
2
+
# alert: its results are scraped into VictoriaMetrics (with a vantage label
3
+
# naming the server) and vmalert alerts on them (GatusEndpointDown).
4
+
let
5
+
inherit (import ./default.nix) alwaysOn;
6
+
7
+
certValid = "[CERTIFICATE_EXPIRATION] > 336h"; # 14 days
8
+
9
+
https = name: url: {
10
+
inherit name url;
11
+
group = "https";
12
+
conditions = [
13
+
"[STATUS] == 200"
14
+
certValid
15
+
];
16
+
};
17
+
18
+
# The primary and the three Hetzner secondaries (configs/dns/knot.nix).
19
+
soa = name: server: {
20
+
inherit name;
21
+
group = "dns";
22
+
url = server;
23
+
dns = {
24
+
query-name = "lassul.us";
25
+
query-type = "SOA";
26
+
};
27
+
conditions = [ "[DNS_RCODE] == NOERROR" ];
28
+
};
29
+
30
+
tls = name: url: {
31
+
inherit name url;
32
+
group = "mail";
33
+
conditions = [
34
+
"[CONNECTED] == true"
35
+
certValid
36
+
];
37
+
};
38
+
39
+
ping = host: {
40
+
name = host;
41
+
group = "retiolum";
42
+
url = "icmp://${host}.r";
43
+
conditions = [ "[CONNECTED] == true" ];
44
+
};
45
+
in
46
+
{
47
+
endpoints = map (e: { interval = "2m"; } // e) (
48
+
[
49
+
(https "lassul.us" "https://lassul.us/")
50
+
(https "matrix" "https://matrix.lassul.us/_matrix/client/versions")
51
+
(https "pad" "https://pad.lassul.us/")
52
+
(https "cache" "https://cache.lassul.us/nix-cache-info")
53
+
(https "niks3" "https://niks3.lassul.us/health")
54
+
(soa "ns1" "95.217.192.59")
55
+
(soa "ns1.first-ns.de" "213.239.242.238")
56
+
(soa "robotns2.second-ns.de" "213.133.100.103")
57
+
(soa "robotns3.second-ns.com" "193.47.99.3")
58
+
(tls "smtps" "tls://mail.lassul.us:465")
59
+
(tls "imaps" "tls://mail.lassul.us:993")
60
+
{
61
+
name = "federation";
62
+
group = "matrix";
63
+
url = "https://federationtester.matrix.org/api/report?server_name=lassul.us";
64
+
interval = "30m";
65
+
conditions = [
66
+
"[STATUS] == 200"
67
+
"[BODY].FederationOK == true"
68
+
];
69
+
}
70
+
]
71
+
++ map ping alwaysOn
72
+
);
73
+
}
+35
configs/monitoring/default.nix
+35
configs/monitoring/default.nix
···
1
+
# Shared facts of the monitoring stack, imported (not as a module) by
2
+
# agent.nix, server.nix, alerts.nix and checks.nix.
3
+
{
4
+
# Every machine writes metrics and logs to all of these (via <name>.r).
5
+
servers = [
6
+
"styx"
7
+
"coaxmetal"
8
+
];
9
+
10
+
# Machines expected to be up all the time. Only these get HostDown alerts
11
+
# and ICMP checks, so laptops can come and go silently. starkstrom has no
12
+
# retiolum route to the servers, so its data never arrives.
13
+
alwaysOn = [
14
+
"neoprism"
15
+
"styx"
16
+
"coaxmetal"
17
+
"acheron"
18
+
"orange"
19
+
"radio"
20
+
"icarus"
21
+
"shodan"
22
+
];
23
+
24
+
ports = {
25
+
victoriametrics = 8428;
26
+
victorialogs = 9428;
27
+
vmalert = 8880;
28
+
vmalert-logs = 8881; # localhost only
29
+
alertmanager = 9093;
30
+
alertmanager-cluster = 9094;
31
+
matrix-receiver = 9095; # localhost only
32
+
gatus = 8080;
33
+
grafana = 3000;
34
+
};
35
+
}
+242
configs/monitoring/server.nix
+242
configs/monitoring/server.nix
···
1
+
# One monitoring server (styx, coaxmetal): VictoriaMetrics and VictoriaLogs
2
+
# receive from the Vector agents, vmalert evaluates alerts/checks into the
3
+
# Alertmanager cluster formed by all servers, which posts to Matrix. Gatus
4
+
# runs the functional checks, Grafana shows it all. Everything listens on
5
+
# retiolum only.
6
+
{
7
+
config,
8
+
lib,
9
+
pkgs,
10
+
self,
11
+
...
12
+
}:
13
+
let
14
+
inherit (import ./default.nix) servers ports;
15
+
alerts = import ./alerts.nix { inherit lib; };
16
+
17
+
name = config.networking.hostName;
18
+
peers = lib.remove name servers;
19
+
# Alertmanager gossips the address it advertises, so it must be the
20
+
# retiolum one, not whatever private address it would pick on its own.
21
+
hosts = self.inputs.kartei.hosts // self.retiolum.hosts;
22
+
retiolumIp = hosts.${name}.nets.retiolum.ip4.addr;
23
+
24
+
receiverConfig = pkgs.writeText "matrix-alertmanager-receiver.json" (
25
+
builtins.toJSON {
26
+
http = {
27
+
address = "127.0.0.1";
28
+
port = ports.matrix-receiver;
29
+
};
30
+
# ${...} is substituted by the receiver from its environment
31
+
# (monitoring-matrix vars).
32
+
matrix = {
33
+
homeserver-url = "https://matrix.lassul.us";
34
+
user-id = "\${MATRIX_USER_ID}";
35
+
access-token = "\${MATRIX_ACCESS_TOKEN}";
36
+
room-mapping.alerts = "\${MATRIX_ROOM_ID}";
37
+
};
38
+
templating.firing-template = ''
39
+
<strong>{{ .Alert.Status | ToUpper }}</strong> {{ .Alert.Labels.alertname }}:
40
+
{{ .Alert.Annotations.summary }}
41
+
'';
42
+
}
43
+
);
44
+
45
+
grafanaVars = config.clan.core.vars.generators.monitoring-grafana.files;
46
+
in
47
+
{
48
+
services.victoriametrics = {
49
+
enable = true;
50
+
listenAddress = ":${toString ports.victoriametrics}";
51
+
retentionPeriod = "90d";
52
+
# Gatus results; the vantage label tells which server saw a failure.
53
+
prometheusConfig.scrape_configs = [
54
+
{
55
+
job_name = "gatus";
56
+
static_configs = [
57
+
{
58
+
targets = [ "127.0.0.1:${toString ports.gatus}" ];
59
+
labels.vantage = name;
60
+
}
61
+
];
62
+
}
63
+
];
64
+
};
65
+
66
+
services.victorialogs = {
67
+
enable = true;
68
+
listenAddress = ":${toString ports.victorialogs}";
69
+
extraOptions = [ "-retentionPeriod=30d" ];
70
+
};
71
+
72
+
services.vmalert.instances = {
73
+
"" = {
74
+
enable = true;
75
+
rules = alerts.metrics;
76
+
settings = {
77
+
"datasource.url" = "http://127.0.0.1:${toString ports.victoriametrics}";
78
+
"notifier.url" = map (s: "http://${s}.r:${toString ports.alertmanager}") servers;
79
+
"httpListenAddr" = ":${toString ports.vmalert}";
80
+
"external.url" = "http://${name}.r:${toString ports.vmalert}";
81
+
};
82
+
};
83
+
logs = {
84
+
enable = true;
85
+
rules = alerts.logs;
86
+
settings = {
87
+
"datasource.url" = "http://127.0.0.1:${toString ports.victorialogs}";
88
+
"notifier.url" = map (s: "http://${s}.r:${toString ports.alertmanager}") servers;
89
+
"httpListenAddr" = "127.0.0.1:${toString ports.vmalert-logs}";
90
+
"rule.defaultRuleType" = "vlogs";
91
+
};
92
+
};
93
+
};
94
+
95
+
services.prometheus.alertmanager = {
96
+
enable = true;
97
+
port = ports.alertmanager;
98
+
webExternalUrl = "http://${name}.r:${toString ports.alertmanager}";
99
+
clusterPeers = map (p: "${p}.r") peers;
100
+
extraFlags = [
101
+
"--cluster.listen-address=[::]:${toString ports.alertmanager-cluster}"
102
+
"--cluster.advertise-address=${retiolumIp}:${toString ports.alertmanager-cluster}"
103
+
];
104
+
configuration = {
105
+
route = {
106
+
receiver = "matrix";
107
+
group_by = [
108
+
"alertname"
109
+
"host"
110
+
];
111
+
repeat_interval = "12h";
112
+
};
113
+
receivers = [
114
+
{
115
+
name = "matrix";
116
+
webhook_configs = [
117
+
{
118
+
url = "http://127.0.0.1:${toString ports.matrix-receiver}/alerts/alerts";
119
+
send_resolved = true;
120
+
}
121
+
];
122
+
}
123
+
];
124
+
};
125
+
};
126
+
127
+
# No NixOS module for it upstream. It exits when the homeserver is
128
+
# unreachable at startup, so keep retrying instead of hitting the start
129
+
# limit (e.g. while neoprism reboots).
130
+
systemd.services.matrix-alertmanager-receiver = {
131
+
description = "Forward Alertmanager alerts to Matrix";
132
+
wantedBy = [ "multi-user.target" ];
133
+
after = [ "network-online.target" ];
134
+
wants = [ "network-online.target" ];
135
+
startLimitIntervalSec = 0;
136
+
serviceConfig = {
137
+
ExecStart = "${lib.getExe pkgs.matrix-alertmanager-receiver} --config-path ${receiverConfig}";
138
+
EnvironmentFile = config.clan.core.vars.generators.monitoring-matrix.files.env.path;
139
+
DynamicUser = true;
140
+
Restart = "always";
141
+
RestartSec = 30;
142
+
};
143
+
};
144
+
145
+
# Shared, so both servers post as the same bot into the same room.
146
+
# Declared identically on every server, as clan requires for shared vars.
147
+
clan.core.vars.generators.monitoring-matrix = {
148
+
share = true;
149
+
files.env = { };
150
+
prompts.user-id.description = "Matrix user id of the alert bot, e.g. @alerts:lassul.us";
151
+
prompts.access-token = {
152
+
description = "Access token of the alert bot";
153
+
type = "hidden";
154
+
};
155
+
prompts.room-id.description = "Matrix room id for alerts (!…:lassul.us); invite the bot first";
156
+
script = ''
157
+
{
158
+
printf 'MATRIX_USER_ID=%s\n' "$(cat "$prompts/user-id")"
159
+
printf 'MATRIX_ACCESS_TOKEN=%s\n' "$(cat "$prompts/access-token")"
160
+
printf 'MATRIX_ROOM_ID=%s\n' "$(cat "$prompts/room-id")"
161
+
} > "$out/env"
162
+
'';
163
+
};
164
+
165
+
services.gatus = {
166
+
enable = true;
167
+
settings = import ./checks.nix // {
168
+
web.port = ports.gatus;
169
+
metrics = true;
170
+
};
171
+
};
172
+
173
+
services.grafana = {
174
+
enable = true;
175
+
declarativePlugins = [ pkgs.grafanaPlugins.victoriametrics-logs-datasource ];
176
+
settings = {
177
+
server = {
178
+
http_addr = "";
179
+
http_port = ports.grafana;
180
+
};
181
+
security = {
182
+
admin_password = "$__file{${grafanaVars.admin-password.path}}";
183
+
secret_key = "$__file{${grafanaVars.secret-key.path}}";
184
+
};
185
+
"auth.anonymous" = {
186
+
enabled = true;
187
+
org_role = "Viewer";
188
+
};
189
+
analytics.reporting_enabled = false;
190
+
};
191
+
provision = {
192
+
enable = true;
193
+
datasources.settings.datasources = [
194
+
{
195
+
name = "VictoriaMetrics";
196
+
uid = "victoriametrics";
197
+
type = "prometheus";
198
+
url = "http://127.0.0.1:${toString ports.victoriametrics}";
199
+
isDefault = true;
200
+
}
201
+
{
202
+
name = "VictoriaLogs";
203
+
uid = "victorialogs";
204
+
type = "victoriametrics-logs-datasource";
205
+
url = "http://127.0.0.1:${toString ports.victorialogs}";
206
+
}
207
+
];
208
+
dashboards.settings.providers = [
209
+
{
210
+
name = "monitoring";
211
+
options.path = ./dashboards;
212
+
}
213
+
];
214
+
};
215
+
};
216
+
217
+
clan.core.vars.generators.monitoring-grafana = {
218
+
files.admin-password.owner = "grafana";
219
+
files.secret-key.owner = "grafana";
220
+
runtimeInputs = [
221
+
pkgs.coreutils
222
+
pkgs.openssl
223
+
];
224
+
script = ''
225
+
openssl rand -hex 16 | tr -d '\n' > "$out/admin-password"
226
+
openssl rand -hex 32 | tr -d '\n' > "$out/secret-key"
227
+
'';
228
+
};
229
+
230
+
networking.firewall.interfaces.retiolum = {
231
+
allowedTCPPorts = with ports; [
232
+
victoriametrics
233
+
victorialogs
234
+
vmalert
235
+
alertmanager
236
+
alertmanager-cluster
237
+
gatus
238
+
grafana
239
+
];
240
+
allowedUDPPorts = [ ports.alertmanager-cluster ];
241
+
};
242
+
}
-194
configs/monitoring/telegraf.nix
-194
configs/monitoring/telegraf.nix
···
1
-
{
2
-
pkgs,
3
-
lib,
4
-
config,
5
-
...
6
-
}:
7
-
# To use this module you also need to allow port 9273 either on the internet or on a vpn interface
8
-
# i.e. networking.firewall.interfaces."vpn0".allowedTCPPorts = [ 9273 ];
9
-
# Example prometheus alert rules:
10
-
# - https://github.com/Mic92/dotfiles/blob/master/nixos/eva/modules/prometheus/alert-rules.nix
11
-
let
12
-
isVM = lib.any (
13
-
mod: mod == "xen-blkfront" || mod == "virtio_console"
14
-
) config.boot.initrd.kernelModules;
15
-
# potentially wrong if the nvme is not used at boot...
16
-
hasNvme = lib.any (m: m == "nvme") config.boot.initrd.availableKernelModules;
17
-
18
-
ipv6DadCheck = pkgs.writeShellScript "ipv6-dad-check" ''
19
-
${pkgs.iproute2}/bin/ip --json addr | \
20
-
${pkgs.jq}/bin/jq -r 'map(.addr_info) | flatten(1) | map(select(.dadfailed == true)) | map(.local) | @text "ipv6_dad_failures count=\(length)i"'
21
-
'';
22
-
23
-
zfsChecks =
24
-
lib.optional (lib.any (fs: fs == "zfs") (lib.attrNames config.boot.supportedFilesystems))
25
-
(
26
-
pkgs.writeScript "zpool-health" ''
27
-
#!${pkgs.gawk}/bin/awk -f
28
-
BEGIN {
29
-
while ("${pkgs.zfs}/bin/zpool status" | getline) {
30
-
if ($1 ~ /pool:/) { printf "zpool_status,name=%s ", $2 }
31
-
if ($1 ~ /state:/) { printf " state=\"%s\",", $2 }
32
-
if ($1 ~ /errors:/) {
33
-
if (index($2, "No")) printf "errors=0i\n"; else printf "errors=%di\n", $2
34
-
}
35
-
}
36
-
}
37
-
''
38
-
);
39
-
40
-
nfsChecks =
41
-
let
42
-
collectHosts =
43
-
shares: fs:
44
-
if
45
-
builtins.elem fs.fsType [
46
-
"nfs"
47
-
"nfs3"
48
-
"nfs4"
49
-
]
50
-
then
51
-
shares
52
-
// (
53
-
let
54
-
# also match ipv6 addresses
55
-
group = builtins.match "\\[?([^\]]+)]?:([^:]+)$" fs.device;
56
-
host = builtins.head group;
57
-
path = builtins.elemAt group 1;
58
-
in
59
-
{
60
-
${host} = (shares.${host} or [ ]) ++ [ path ];
61
-
}
62
-
)
63
-
else
64
-
shares;
65
-
nfsHosts = lib.foldl collectHosts { } (builtins.attrValues config.fileSystems);
66
-
in
67
-
lib.mapAttrsToList (
68
-
host: args:
69
-
(pkgs.writeScript "nfs-health" ''
70
-
#!${pkgs.gawk}/bin/awk -f
71
-
BEGIN {
72
-
for (i = 2; i < ARGC; i++) {
73
-
mounts[ARGV[i]] = 1
74
-
}
75
-
while ("${pkgs.nfs-utils}/bin/showmount -e " ARGV[1] | getline) {
76
-
if (NR == 1) { continue }
77
-
if (mounts[$1] == 1) {
78
-
printf "nfs_export,host=%s,path=%s present=1\n", ARGV[1], $1
79
-
}
80
-
delete mounts[$1]
81
-
}
82
-
for (mount in mounts) {
83
-
printf "nfs_export,host=%s,path=%s present=0\n", ARGV[1], $1
84
-
}
85
-
}
86
-
'')
87
-
+ " ${host} ${builtins.concatStringsSep " " args}"
88
-
) nfsHosts;
89
-
90
-
in
91
-
{
92
-
93
-
systemd.services.telegraf.path = lib.optional (!isVM && hasNvme) pkgs.nvme-cli;
94
-
95
-
services.telegraf = {
96
-
enable = true;
97
-
extraConfig = {
98
-
agent.interval = "60s";
99
-
inputs = {
100
-
prometheus.metric_version = 2;
101
-
kernel_vmstat = { };
102
-
nginx.urls = lib.mkIf config.services.nginx.statusPage [
103
-
"http://localhost/nginx_status"
104
-
];
105
-
smart = lib.mkIf (!isVM) {
106
-
path_smartctl = pkgs.writeShellScript "smartctl" ''
107
-
exec /run/wrappers/bin/sudo ${pkgs.smartmontools}/bin/smartctl "$@"
108
-
'';
109
-
};
110
-
system = { };
111
-
mem = { };
112
-
file = [
113
-
{
114
-
data_format = "influx";
115
-
file_tag = "name";
116
-
files = [ "/var/log/telegraf/*" ];
117
-
}
118
-
]
119
-
++ lib.optional (lib.any (fs: fs == "ext4") (lib.attrNames config.boot.supportedFilesystems)) {
120
-
name_override = "ext4_errors";
121
-
files = [ "/sys/fs/ext4/*/errors_count" ];
122
-
data_format = "value";
123
-
};
124
-
exec = [
125
-
{
126
-
## Commands array
127
-
commands = [ ipv6DadCheck ] ++ zfsChecks ++ nfsChecks;
128
-
data_format = "influx";
129
-
}
130
-
];
131
-
systemd_units = { };
132
-
swap = { };
133
-
disk.tagdrop = {
134
-
fstype = [
135
-
"tmpfs"
136
-
"ramfs"
137
-
"devtmpfs"
138
-
"devfs"
139
-
"iso9660"
140
-
"overlay"
141
-
"aufs"
142
-
"squashfs"
143
-
];
144
-
device = [
145
-
"rpc_pipefs"
146
-
"lxcfs"
147
-
"nsfs"
148
-
"borgfs"
149
-
];
150
-
};
151
-
diskio = { };
152
-
zfs = {
153
-
poolMetrics = true;
154
-
};
155
-
}
156
-
//
157
-
lib.optionalAttrs
158
-
(
159
-
if lib.versionAtLeast (lib.versions.majorMinor lib.version) "23.11" then
160
-
config.boot.swraid.enable
161
-
else
162
-
config.boot.initrd.services.swraid.enable
163
-
)
164
-
{
165
-
mdstat = { };
166
-
};
167
-
outputs.prometheus_client = {
168
-
listen = ":9273";
169
-
metric_version = 2;
170
-
};
171
-
};
172
-
};
173
-
security.sudo.extraRules = lib.mkIf (!isVM) [
174
-
{
175
-
users = [ "telegraf" ];
176
-
commands = [
177
-
{
178
-
command = "${pkgs.smartmontools}/bin/smartctl";
179
-
options = [ "NOPASSWD" ];
180
-
}
181
-
];
182
-
}
183
-
];
184
-
networking.firewall.allowedTCPPorts = [ 9273 ];
185
-
186
-
# avoid logging sudo use
187
-
security.sudo.configFile = ''
188
-
Defaults:telegraf !syslog,!pam_session
189
-
'';
190
-
# create dummy file to avoid telegraf errors
191
-
systemd.tmpfiles.rules = [
192
-
"f /var/log/telegraf/dummy 0444 root root - -"
193
-
];
194
-
}
+1
machines/coaxmetal/config.nix
+1
machines/coaxmetal/config.nix
-10
machines/neoprism/backup.nix
-10
machines/neoprism/backup.nix
···
1
1
{ config, pkgs, ... }:
2
2
{
3
-
systemd.services.borgbackup-job-hetzner.serviceConfig.ReadWritePaths = [
4
-
"/var/log/telegraf"
5
-
];
6
-
7
3
clan.core.vars.generators.borgbackup = {
8
4
files."borgbackup.ssh.id25519" = { };
9
5
files."borgbackup.ssh.id25519.pub".secret = false;
···
92
88
# Dump timeful polls (when.lassul.us) from MongoDB
93
89
${pkgs.mongodb-tools}/bin/mongodump --uri=mongodb://127.0.0.1:27017 \
94
90
--db=schej-it --gzip --archive=/var/backup/timeful.archive.gz
95
-
'';
96
-
97
-
postHook = ''
98
-
cat > /var/log/telegraf/borgbackup-job-hetzner.service <<EOF
99
-
task,frequency=daily last_run=$(date +%s)i,state="$([[ $exitStatus == 0 ]] && echo ok || echo fail)"
100
-
EOF
101
91
'';
102
92
103
93
prune.keep = {
-1
machines/neoprism/config.nix
-1
machines/neoprism/config.nix
+1
machines/styx/config.nix
+1
machines/styx/config.nix
+51
configs/monitoring/dashboards/gatus.json
+51
configs/monitoring/dashboards/gatus.json
···
1
+
{
2
+
"uid": "gatus",
3
+
"title": "Gatus",
4
+
"schemaVersion": 39,
5
+
"time": { "from": "now-24h", "to": "now" },
6
+
"refresh": "1m",
7
+
"panels": [
8
+
{
9
+
"id": 1,
10
+
"type": "state-timeline",
11
+
"title": "Endpoint success",
12
+
"gridPos": { "x": 0, "y": 0, "w": 24, "h": 16 },
13
+
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
14
+
"fieldConfig": {
15
+
"defaults": {
16
+
"mappings": [
17
+
{
18
+
"type": "value",
19
+
"options": {
20
+
"0": { "text": "down", "color": "red" },
21
+
"1": { "text": "up", "color": "green" }
22
+
}
23
+
}
24
+
]
25
+
}
26
+
},
27
+
"targets": [
28
+
{
29
+
"refId": "A",
30
+
"expr": "gatus_results_endpoint_success",
31
+
"legendFormat": "{{vantage}} {{group}}/{{name}}"
32
+
}
33
+
]
34
+
},
35
+
{
36
+
"id": 2,
37
+
"type": "timeseries",
38
+
"title": "Response time",
39
+
"gridPos": { "x": 0, "y": 16, "w": 24, "h": 8 },
40
+
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
41
+
"fieldConfig": { "defaults": { "unit": "s" } },
42
+
"targets": [
43
+
{
44
+
"refId": "A",
45
+
"expr": "gatus_results_duration_seconds",
46
+
"legendFormat": "{{vantage}} {{group}}/{{name}}"
47
+
}
48
+
]
49
+
}
50
+
]
51
+
}
+46
configs/monitoring/dashboards/logs.json
+46
configs/monitoring/dashboards/logs.json
···
1
+
{
2
+
"uid": "logs",
3
+
"title": "Logs explorer",
4
+
"schemaVersion": 39,
5
+
"time": { "from": "now-1h", "to": "now" },
6
+
"templating": {
7
+
"list": [
8
+
{
9
+
"name": "host",
10
+
"type": "custom",
11
+
"query": "*",
12
+
"current": { "text": "*", "value": "*" }
13
+
},
14
+
{
15
+
"name": "unit",
16
+
"type": "custom",
17
+
"query": "*",
18
+
"current": { "text": "*", "value": "*" }
19
+
},
20
+
{
21
+
"name": "search",
22
+
"type": "textbox",
23
+
"query": "*",
24
+
"current": { "text": "*", "value": "*" }
25
+
}
26
+
]
27
+
},
28
+
"panels": [
29
+
{
30
+
"id": 1,
31
+
"type": "logs",
32
+
"title": "Journal",
33
+
"gridPos": { "x": 0, "y": 0, "w": 24, "h": 24 },
34
+
"datasource": { "type": "victoriametrics-logs-datasource", "uid": "victorialogs" },
35
+
"options": { "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true },
36
+
"targets": [
37
+
{
38
+
"refId": "A",
39
+
"queryType": "instant",
40
+
"expr": "host:$host unit:$unit $search",
41
+
"maxLines": 1000
42
+
}
43
+
]
44
+
}
45
+
]
46
+
}
+134
configs/monitoring/dashboards/node.json
+134
configs/monitoring/dashboards/node.json
···
1
+
{
2
+
"uid": "node",
3
+
"title": "Node overview",
4
+
"schemaVersion": 39,
5
+
"time": { "from": "now-24h", "to": "now" },
6
+
"refresh": "1m",
7
+
"templating": {
8
+
"list": [
9
+
{
10
+
"name": "host",
11
+
"type": "query",
12
+
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
13
+
"query": { "query": "label_values(host_uptime, host)", "refId": "host" },
14
+
"refresh": 2,
15
+
"includeAll": true,
16
+
"multi": true,
17
+
"current": { "text": "All", "value": "$__all" }
18
+
}
19
+
]
20
+
},
21
+
"panels": [
22
+
{
23
+
"id": 1,
24
+
"type": "timeseries",
25
+
"title": "CPU used",
26
+
"gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 },
27
+
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
28
+
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1 } },
29
+
"targets": [
30
+
{
31
+
"refId": "A",
32
+
"expr": "1 - avg by (host) (rate(host_cpu_seconds_total{mode=\"idle\", host=~\"$host\"}[$__rate_interval]))",
33
+
"legendFormat": "{{host}}"
34
+
}
35
+
]
36
+
},
37
+
{
38
+
"id": 2,
39
+
"type": "timeseries",
40
+
"title": "Memory used",
41
+
"gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 },
42
+
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
43
+
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1 } },
44
+
"targets": [
45
+
{
46
+
"refId": "A",
47
+
"expr": "1 - host_memory_available_bytes{host=~\"$host\"} / host_memory_total_bytes{host=~\"$host\"}",
48
+
"legendFormat": "{{host}}"
49
+
}
50
+
]
51
+
},
52
+
{
53
+
"id": 3,
54
+
"type": "timeseries",
55
+
"title": "Load (1m)",
56
+
"gridPos": { "x": 0, "y": 8, "w": 12, "h": 8 },
57
+
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
58
+
"targets": [
59
+
{ "refId": "A", "expr": "host_load1{host=~\"$host\"}", "legendFormat": "{{host}}" }
60
+
]
61
+
},
62
+
{
63
+
"id": 4,
64
+
"type": "bargauge",
65
+
"title": "Filesystems used",
66
+
"gridPos": { "x": 12, "y": 8, "w": 12, "h": 8 },
67
+
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
68
+
"fieldConfig": {
69
+
"defaults": {
70
+
"unit": "percentunit",
71
+
"min": 0,
72
+
"max": 1,
73
+
"thresholds": {
74
+
"mode": "absolute",
75
+
"steps": [
76
+
{ "color": "green", "value": null },
77
+
{ "color": "orange", "value": 0.8 },
78
+
{ "color": "red", "value": 0.9 }
79
+
]
80
+
}
81
+
}
82
+
},
83
+
"options": { "displayMode": "basic", "orientation": "horizontal" },
84
+
"targets": [
85
+
{
86
+
"refId": "A",
87
+
"expr": "max by (host, mountpoint) (host_filesystem_used_ratio{host=~\"$host\"})",
88
+
"legendFormat": "{{host}} {{mountpoint}}",
89
+
"instant": true
90
+
}
91
+
]
92
+
},
93
+
{
94
+
"id": 5,
95
+
"type": "timeseries",
96
+
"title": "Network",
97
+
"gridPos": { "x": 0, "y": 16, "w": 12, "h": 8 },
98
+
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
99
+
"fieldConfig": { "defaults": { "unit": "Bps" } },
100
+
"targets": [
101
+
{
102
+
"refId": "A",
103
+
"expr": "sum by (host) (rate(host_network_receive_bytes_total{host=~\"$host\", device!=\"lo\"}[$__rate_interval]))",
104
+
"legendFormat": "{{host}} rx"
105
+
},
106
+
{
107
+
"refId": "B",
108
+
"expr": "-sum by (host) (rate(host_network_transmit_bytes_total{host=~\"$host\", device!=\"lo\"}[$__rate_interval]))",
109
+
"legendFormat": "{{host}} tx"
110
+
}
111
+
]
112
+
},
113
+
{
114
+
"id": 6,
115
+
"type": "timeseries",
116
+
"title": "Disk I/O",
117
+
"gridPos": { "x": 12, "y": 16, "w": 12, "h": 8 },
118
+
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
119
+
"fieldConfig": { "defaults": { "unit": "Bps" } },
120
+
"targets": [
121
+
{
122
+
"refId": "A",
123
+
"expr": "sum by (host) (rate(host_disk_read_bytes_total{host=~\"$host\"}[$__rate_interval]))",
124
+
"legendFormat": "{{host}} read"
125
+
},
126
+
{
127
+
"refId": "B",
128
+
"expr": "-sum by (host) (rate(host_disk_written_bytes_total{host=~\"$host\"}[$__rate_interval]))",
129
+
"legendFormat": "{{host}} write"
130
+
}
131
+
]
132
+
}
133
+
]
134
+
}