This repository has no description
7.4 kB
242 lines
1# One monitoring server (styx, coaxmetal): VictoriaMetrics and VictoriaLogs
2# receive from the Vector agents, vmalert evaluates alerts/checks into the
3# Alertmanager cluster formed by all servers, which posts to Matrix. Gatus
4# runs the functional checks, Grafana shows it all. Everything listens on
5# retiolum only.
6{
7 config,
8 lib,
9 pkgs,
10 self,
11 ...
12}:
13let
14 inherit (import ./default.nix) servers ports;
15 alerts = import ./alerts.nix { inherit lib; };
16
17 name = config.networking.hostName;
18 peers = lib.remove name servers;
19 # Alertmanager gossips the address it advertises, so it must be the
20 # retiolum one, not whatever private address it would pick on its own.
21 hosts = self.inputs.kartei.hosts // self.retiolum.hosts;
22 retiolumIp = hosts.${name}.nets.retiolum.ip4.addr;
23
24 receiverConfig = pkgs.writeText "matrix-alertmanager-receiver.json" (
25 builtins.toJSON {
26 http = {
27 address = "127.0.0.1";
28 port = ports.matrix-receiver;
29 };
30 # ${...} is substituted by the receiver from its environment
31 # (monitoring-matrix vars).
32 matrix = {
33 homeserver-url = "https://matrix.lassul.us";
34 user-id = "\${MATRIX_USER_ID}";
35 access-token = "\${MATRIX_ACCESS_TOKEN}";
36 room-mapping.alerts = "\${MATRIX_ROOM_ID}";
37 };
38 templating.firing-template = ''
39 <strong>{{ .Alert.Status | ToUpper }}</strong> {{ .Alert.Labels.alertname }}:
40 {{ .Alert.Annotations.summary }}
41 '';
42 }
43 );
44
45 grafanaVars = config.clan.core.vars.generators.monitoring-grafana.files;
46in
47{
48 services.victoriametrics = {
49 enable = true;
50 listenAddress = ":${toString ports.victoriametrics}";
51 retentionPeriod = "90d";
52 # Gatus results; the vantage label tells which server saw a failure.
53 prometheusConfig.scrape_configs = [
54 {
55 job_name = "gatus";
56 static_configs = [
57 {
58 targets = [ "127.0.0.1:${toString ports.gatus}" ];
59 labels.vantage = name;
60 }
61 ];
62 }
63 ];
64 };
65
66 services.victorialogs = {
67 enable = true;
68 listenAddress = ":${toString ports.victorialogs}";
69 extraOptions = [ "-retentionPeriod=30d" ];
70 };
71
72 services.vmalert.instances = {
73 "" = {
74 enable = true;
75 rules = alerts.metrics;
76 settings = {
77 "datasource.url" = "http://127.0.0.1:${toString ports.victoriametrics}";
78 "notifier.url" = map (s: "http://${s}.r:${toString ports.alertmanager}") servers;
79 "httpListenAddr" = ":${toString ports.vmalert}";
80 "external.url" = "http://${name}.r:${toString ports.vmalert}";
81 };
82 };
83 logs = {
84 enable = true;
85 rules = alerts.logs;
86 settings = {
87 "datasource.url" = "http://127.0.0.1:${toString ports.victorialogs}";
88 "notifier.url" = map (s: "http://${s}.r:${toString ports.alertmanager}") servers;
89 "httpListenAddr" = "127.0.0.1:${toString ports.vmalert-logs}";
90 "rule.defaultRuleType" = "vlogs";
91 };
92 };
93 };
94
95 services.prometheus.alertmanager = {
96 enable = true;
97 port = ports.alertmanager;
98 webExternalUrl = "http://${name}.r:${toString ports.alertmanager}";
99 clusterPeers = map (p: "${p}.r") peers;
100 extraFlags = [
101 "--cluster.listen-address=[::]:${toString ports.alertmanager-cluster}"
102 "--cluster.advertise-address=${retiolumIp}:${toString ports.alertmanager-cluster}"
103 ];
104 configuration = {
105 route = {
106 receiver = "matrix";
107 group_by = [
108 "alertname"
109 "host"
110 ];
111 repeat_interval = "12h";
112 };
113 receivers = [
114 {
115 name = "matrix";
116 webhook_configs = [
117 {
118 url = "http://127.0.0.1:${toString ports.matrix-receiver}/alerts/alerts";
119 send_resolved = true;
120 }
121 ];
122 }
123 ];
124 };
125 };
126
127 # No NixOS module for it upstream. It exits when the homeserver is
128 # unreachable at startup, so keep retrying instead of hitting the start
129 # limit (e.g. while neoprism reboots).
130 systemd.services.matrix-alertmanager-receiver = {
131 description = "Forward Alertmanager alerts to Matrix";
132 wantedBy = [ "multi-user.target" ];
133 after = [ "network-online.target" ];
134 wants = [ "network-online.target" ];
135 startLimitIntervalSec = 0;
136 serviceConfig = {
137 ExecStart = "${lib.getExe pkgs.matrix-alertmanager-receiver} --config-path ${receiverConfig}";
138 EnvironmentFile = config.clan.core.vars.generators.monitoring-matrix.files.env.path;
139 DynamicUser = true;
140 Restart = "always";
141 RestartSec = 30;
142 };
143 };
144
145 # Shared, so both servers post as the same bot into the same room.
146 # Declared identically on every server, as clan requires for shared vars.
147 clan.core.vars.generators.monitoring-matrix = {
148 share = true;
149 files.env = { };
150 prompts.user-id.description = "Matrix user id of the alert bot, e.g. @alerts:lassul.us";
151 prompts.access-token = {
152 description = "Access token of the alert bot";
153 type = "hidden";
154 };
155 prompts.room-id.description = "Matrix room id for alerts (!…:lassul.us); invite the bot first";
156 script = ''
157 {
158 printf 'MATRIX_USER_ID=%s\n' "$(cat "$prompts/user-id")"
159 printf 'MATRIX_ACCESS_TOKEN=%s\n' "$(cat "$prompts/access-token")"
160 printf 'MATRIX_ROOM_ID=%s\n' "$(cat "$prompts/room-id")"
161 } > "$out/env"
162 '';
163 };
164
165 services.gatus = {
166 enable = true;
167 settings = import ./checks.nix // {
168 web.port = ports.gatus;
169 metrics = true;
170 };
171 };
172
173 services.grafana = {
174 enable = true;
175 declarativePlugins = [ pkgs.grafanaPlugins.victoriametrics-logs-datasource ];
176 settings = {
177 server = {
178 http_addr = "";
179 http_port = ports.grafana;
180 };
181 security = {
182 admin_password = "$__file{${grafanaVars.admin-password.path}}";
183 secret_key = "$__file{${grafanaVars.secret-key.path}}";
184 };
185 "auth.anonymous" = {
186 enabled = true;
187 org_role = "Viewer";
188 };
189 analytics.reporting_enabled = false;
190 };
191 provision = {
192 enable = true;
193 datasources.settings.datasources = [
194 {
195 name = "VictoriaMetrics";
196 uid = "victoriametrics";
197 type = "prometheus";
198 url = "http://127.0.0.1:${toString ports.victoriametrics}";
199 isDefault = true;
200 }
201 {
202 name = "VictoriaLogs";
203 uid = "victorialogs";
204 type = "victoriametrics-logs-datasource";
205 url = "http://127.0.0.1:${toString ports.victorialogs}";
206 }
207 ];
208 dashboards.settings.providers = [
209 {
210 name = "monitoring";
211 options.path = ./dashboards;
212 }
213 ];
214 };
215 };
216
217 clan.core.vars.generators.monitoring-grafana = {
218 files.admin-password.owner = "grafana";
219 files.secret-key.owner = "grafana";
220 runtimeInputs = [
221 pkgs.coreutils
222 pkgs.openssl
223 ];
224 script = ''
225 openssl rand -hex 16 | tr -d '\n' > "$out/admin-password"
226 openssl rand -hex 32 | tr -d '\n' > "$out/secret-key"
227 '';
228 };
229
230 networking.firewall.interfaces.retiolum = {
231 allowedTCPPorts = with ports; [
232 victoriametrics
233 victorialogs
234 vmalert
235 alertmanager
236 alertmanager-cluster
237 gatus
238 grafana
239 ];
240 allowedUDPPorts = [ ports.alertmanager-cluster ];
241 };
242}