This repository has no description
7.5 kB
245 lines
1# One monitoring server (styx, coaxmetal): VictoriaMetrics and VictoriaLogs
2# receive from the Vector agents, vmalert evaluates alerts/checks into the
3# Alertmanager cluster formed by all servers, which posts to Matrix. Gatus
4# runs the functional checks, Grafana shows it all. Everything listens on
5# retiolum only.
6{
7 config,
8 lib,
9 pkgs,
10 self,
11 ...
12}:
13let
14 inherit (import ./default.nix) servers ports;
15 alerts = import ./alerts.nix { inherit lib; };
16
17 name = config.networking.hostName;
18 peers = lib.remove name servers;
19 # Alertmanager gossips the address it advertises, so it must be the
20 # retiolum one, not whatever private address it would pick on its own.
21 hosts = self.inputs.kartei.hosts // self.retiolum.hosts;
22 retiolumIp = hosts.${name}.nets.retiolum.ip4.addr;
23
24 receiverConfig = pkgs.writeText "matrix-alertmanager-receiver.json" (
25 builtins.toJSON {
26 http = {
27 address = "127.0.0.1";
28 port = ports.matrix-receiver;
29 };
30 # ${...} is substituted by the receiver from its environment
31 # (monitoring-matrix vars).
32 matrix = {
33 homeserver-url = "https://matrix.lassul.us";
34 user-id = "\${MATRIX_USER_ID}";
35 access-token = "\${MATRIX_ACCESS_TOKEN}";
36 room-mapping.alerts = "\${MATRIX_ROOM_ID}";
37 };
38 templating.firing-template = ''
39 {{ if eq .Alert.Labels.severity "critical" }}🚨 {{ end }}<strong>{{ .Alert.Status | ToUpper }}</strong> {{ .Alert.Labels.alertname }}:
40 {{ .Alert.Annotations.summary }}
41 '';
42 }
43 );
44
45 grafanaVars = config.clan.core.vars.generators.monitoring-grafana.files;
46in
47{
48 services.victoriametrics = {
49 enable = true;
50 listenAddress = ":${toString ports.victoriametrics}";
51 retentionPeriod = "90d";
52 # Gatus results; the vantage label tells which server saw a failure.
53 prometheusConfig.scrape_configs = [
54 {
55 job_name = "gatus";
56 static_configs = [
57 {
58 targets = [ "127.0.0.1:${toString ports.gatus}" ];
59 labels.vantage = name;
60 }
61 ];
62 }
63 ];
64 };
65
66 services.victorialogs = {
67 enable = true;
68 listenAddress = ":${toString ports.victorialogs}";
69 extraOptions = [ "-retentionPeriod=30d" ];
70 };
71
72 services.vmalert.instances = {
73 "" = {
74 enable = true;
75 rules = alerts.metrics;
76 settings = {
77 "datasource.url" = "http://127.0.0.1:${toString ports.victoriametrics}";
78 "notifier.url" = map (s: "http://${s}.r:${toString ports.alertmanager}") servers;
79 "httpListenAddr" = ":${toString ports.vmalert}";
80 "external.url" = "http://${name}.r:${toString ports.vmalert}";
81 };
82 };
83 logs = {
84 enable = true;
85 rules = alerts.logs;
86 settings = {
87 "datasource.url" = "http://127.0.0.1:${toString ports.victorialogs}";
88 "notifier.url" = map (s: "http://${s}.r:${toString ports.alertmanager}") servers;
89 "httpListenAddr" = "127.0.0.1:${toString ports.vmalert-logs}";
90 "rule.defaultRuleType" = "vlogs";
91 };
92 };
93 };
94
95 services.prometheus.alertmanager = {
96 enable = true;
97 port = ports.alertmanager;
98 webExternalUrl = "http://${name}.r:${toString ports.alertmanager}";
99 clusterPeers = map (p: "${p}.r") peers;
100 extraFlags = [
101 "--cluster.listen-address=[::]:${toString ports.alertmanager-cluster}"
102 "--cluster.advertise-address=${retiolumIp}:${toString ports.alertmanager-cluster}"
103 ];
104 configuration = {
105 route = {
106 receiver = "matrix";
107 group_by = [
108 "alertname"
109 "host"
110 ];
111 repeat_interval = "12h";
112 };
113 receivers = [
114 {
115 name = "matrix";
116 webhook_configs = [
117 {
118 url = "http://127.0.0.1:${toString ports.matrix-receiver}/alerts/alerts";
119 send_resolved = true;
120 }
121 ];
122 }
123 ];
124 };
125 };
126
127 # No NixOS module for it upstream. It exits when the homeserver is
128 # unreachable at startup, so keep retrying instead of hitting the start
129 # limit (e.g. while neoprism reboots).
130 systemd.services.matrix-alertmanager-receiver = {
131 description = "Forward Alertmanager alerts to Matrix";
132 wantedBy = [ "multi-user.target" ];
133 after = [ "network-online.target" ];
134 wants = [ "network-online.target" ];
135 startLimitIntervalSec = 0;
136 serviceConfig = {
137 ExecStart = "${lib.getExe pkgs.matrix-alertmanager-receiver} --config-path ${receiverConfig}";
138 EnvironmentFile = config.clan.core.vars.generators.monitoring-matrix.files.env.path;
139 DynamicUser = true;
140 Restart = "always";
141 RestartSec = 30;
142 };
143 };
144
145 # Shared, so both servers post as the same bot into the same room.
146 # Declared identically on every server, as clan requires for shared vars.
147 clan.core.vars.generators.monitoring-matrix = {
148 share = true;
149 files.env = { };
150 prompts.user-id.description = "Matrix user id of the alert bot, e.g. @alerts:lassul.us";
151 prompts.access-token = {
152 description = "Access token of the alert bot";
153 type = "hidden";
154 };
155 prompts.room-id.description = "Matrix room id for alerts (!…:lassul.us); invite the bot first";
156 script = ''
157 {
158 printf 'MATRIX_USER_ID=%s\n' "$(cat "$prompts/user-id")"
159 printf 'MATRIX_ACCESS_TOKEN=%s\n' "$(cat "$prompts/access-token")"
160 printf 'MATRIX_ROOM_ID=%s\n' "$(cat "$prompts/room-id")"
161 } > "$out/env"
162 '';
163 };
164
165 services.gatus = {
166 enable = true;
167 settings = import ./checks.nix // {
168 web.port = ports.gatus;
169 metrics = true;
170 };
171 };
172
173 services.grafana = {
174 enable = true;
175 declarativePlugins = [
176 pkgs.grafanaPlugins.victoriametrics-logs-datasource
177 pkgs.grafanaPlugins.victoriametrics-metrics-datasource
178 ];
179 settings = {
180 server = {
181 http_addr = "";
182 http_port = ports.grafana;
183 };
184 security = {
185 admin_password = "$__file{${grafanaVars.admin-password.path}}";
186 secret_key = "$__file{${grafanaVars.secret-key.path}}";
187 };
188 "auth.anonymous" = {
189 enabled = true;
190 org_role = "Viewer";
191 };
192 analytics.reporting_enabled = false;
193 };
194 provision = {
195 enable = true;
196 datasources.settings.datasources = [
197 {
198 name = "VictoriaMetrics";
199 uid = "victoriametrics";
200 type = "victoriametrics-metrics-datasource";
201 url = "http://127.0.0.1:${toString ports.victoriametrics}";
202 isDefault = true;
203 }
204 {
205 name = "VictoriaLogs";
206 uid = "victorialogs";
207 type = "victoriametrics-logs-datasource";
208 url = "http://127.0.0.1:${toString ports.victorialogs}";
209 }
210 ];
211 dashboards.settings.providers = [
212 {
213 name = "monitoring";
214 options.path = ./dashboards;
215 }
216 ];
217 };
218 };
219
220 clan.core.vars.generators.monitoring-grafana = {
221 files.admin-password.owner = "grafana";
222 files.secret-key.owner = "grafana";
223 runtimeInputs = [
224 pkgs.coreutils
225 pkgs.openssl
226 ];
227 script = ''
228 openssl rand -hex 16 | tr -d '\n' > "$out/admin-password"
229 openssl rand -hex 32 | tr -d '\n' > "$out/secret-key"
230 '';
231 };
232
233 networking.firewall.interfaces.retiolum = {
234 allowedTCPPorts = with ports; [
235 victoriametrics
236 victorialogs
237 vmalert
238 alertmanager
239 alertmanager-cluster
240 gatus
241 grafana
242 ];
243 allowedUDPPorts = [ ports.alertmanager-cluster ];
244 };
245}