This repository has no description
7.5 kB
244 lines
1# One monitoring server (styx, coaxmetal): VictoriaMetrics and VictoriaLogs
2# receive from the Vector agents, vmalert evaluates alerts/checks into the
3# Alertmanager cluster formed by all servers, which posts to Matrix. Gatus
4# runs the functional checks, Grafana shows it all. Everything listens on
5# retiolum only.
6{
7 config,
8 lib,
9 pkgs,
10 self,
11 ...
12}:
13let
14 inherit (import ./default.nix) servers ports;
15 alerts = import ./alerts.nix { inherit lib; };
16
17 name = config.networking.hostName;
18 peers = lib.remove name servers;
19 # Alertmanager gossips the address it advertises, so it must be the
20 # retiolum one, not whatever private address it would pick on its own.
21 retiolumIp = self.inputs.kartei.hosts.${name}.nets.retiolum.ip4.addr;
22
23 receiverConfig = pkgs.writeText "matrix-alertmanager-receiver.json" (
24 builtins.toJSON {
25 http = {
26 address = "127.0.0.1";
27 port = ports.matrix-receiver;
28 };
29 # ${...} is substituted by the receiver from its environment
30 # (monitoring-matrix vars).
31 matrix = {
32 homeserver-url = "https://matrix.lassul.us";
33 user-id = "\${MATRIX_USER_ID}";
34 access-token = "\${MATRIX_ACCESS_TOKEN}";
35 room-mapping.alerts = "\${MATRIX_ROOM_ID}";
36 };
37 templating.firing-template = ''
38 {{ if eq .Alert.Labels.severity "critical" }}🚨 {{ end }}<strong>{{ .Alert.Status | ToUpper }}</strong> {{ .Alert.Labels.alertname }}:
39 {{ .Alert.Annotations.summary }}
40 '';
41 }
42 );
43
44 grafanaVars = config.clan.core.vars.generators.monitoring-grafana.files;
45in
46{
47 services.victoriametrics = {
48 enable = true;
49 listenAddress = ":${toString ports.victoriametrics}";
50 retentionPeriod = "90d";
51 # Gatus results; the vantage label tells which server saw a failure.
52 prometheusConfig.scrape_configs = [
53 {
54 job_name = "gatus";
55 static_configs = [
56 {
57 targets = [ "127.0.0.1:${toString ports.gatus}" ];
58 labels.vantage = name;
59 }
60 ];
61 }
62 ];
63 };
64
65 services.victorialogs = {
66 enable = true;
67 listenAddress = ":${toString ports.victorialogs}";
68 extraOptions = [ "-retentionPeriod=30d" ];
69 };
70
71 services.vmalert.instances = {
72 "" = {
73 enable = true;
74 rules = alerts.metrics;
75 settings = {
76 "datasource.url" = "http://127.0.0.1:${toString ports.victoriametrics}";
77 "notifier.url" = map (s: "http://${s}.r:${toString ports.alertmanager}") servers;
78 "httpListenAddr" = ":${toString ports.vmalert}";
79 "external.url" = "http://${name}.r:${toString ports.vmalert}";
80 };
81 };
82 logs = {
83 enable = true;
84 rules = alerts.logs;
85 settings = {
86 "datasource.url" = "http://127.0.0.1:${toString ports.victorialogs}";
87 "notifier.url" = map (s: "http://${s}.r:${toString ports.alertmanager}") servers;
88 "httpListenAddr" = "127.0.0.1:${toString ports.vmalert-logs}";
89 "rule.defaultRuleType" = "vlogs";
90 };
91 };
92 };
93
94 services.prometheus.alertmanager = {
95 enable = true;
96 port = ports.alertmanager;
97 webExternalUrl = "http://${name}.r:${toString ports.alertmanager}";
98 clusterPeers = map (p: "${p}.r") peers;
99 extraFlags = [
100 "--cluster.listen-address=[::]:${toString ports.alertmanager-cluster}"
101 "--cluster.advertise-address=${retiolumIp}:${toString ports.alertmanager-cluster}"
102 ];
103 configuration = {
104 route = {
105 receiver = "matrix";
106 group_by = [
107 "alertname"
108 "host"
109 ];
110 repeat_interval = "12h";
111 };
112 receivers = [
113 {
114 name = "matrix";
115 webhook_configs = [
116 {
117 url = "http://127.0.0.1:${toString ports.matrix-receiver}/alerts/alerts";
118 send_resolved = true;
119 }
120 ];
121 }
122 ];
123 };
124 };
125
126 # No NixOS module for it upstream. It exits when the homeserver is
127 # unreachable at startup, so keep retrying instead of hitting the start
128 # limit (e.g. while neoprism reboots).
129 systemd.services.matrix-alertmanager-receiver = {
130 description = "Forward Alertmanager alerts to Matrix";
131 wantedBy = [ "multi-user.target" ];
132 after = [ "network-online.target" ];
133 wants = [ "network-online.target" ];
134 startLimitIntervalSec = 0;
135 serviceConfig = {
136 ExecStart = "${lib.getExe pkgs.matrix-alertmanager-receiver} --config-path ${receiverConfig}";
137 EnvironmentFile = config.clan.core.vars.generators.monitoring-matrix.files.env.path;
138 DynamicUser = true;
139 Restart = "always";
140 RestartSec = 30;
141 };
142 };
143
144 # Shared, so both servers post as the same bot into the same room.
145 # Declared identically on every server, as clan requires for shared vars.
146 clan.core.vars.generators.monitoring-matrix = {
147 share = true;
148 files.env = { };
149 prompts.user-id.description = "Matrix user id of the alert bot, e.g. @alerts:lassul.us";
150 prompts.access-token = {
151 description = "Access token of the alert bot";
152 type = "hidden";
153 };
154 prompts.room-id.description = "Matrix room id for alerts (!…:lassul.us); invite the bot first";
155 script = ''
156 {
157 printf 'MATRIX_USER_ID=%s\n' "$(cat "$prompts/user-id")"
158 printf 'MATRIX_ACCESS_TOKEN=%s\n' "$(cat "$prompts/access-token")"
159 printf 'MATRIX_ROOM_ID=%s\n' "$(cat "$prompts/room-id")"
160 } > "$out/env"
161 '';
162 };
163
164 services.gatus = {
165 enable = true;
166 settings = import ./checks.nix // {
167 web.port = ports.gatus;
168 metrics = true;
169 };
170 };
171
172 services.grafana = {
173 enable = true;
174 declarativePlugins = [
175 pkgs.grafanaPlugins.victoriametrics-logs-datasource
176 pkgs.grafanaPlugins.victoriametrics-metrics-datasource
177 ];
178 settings = {
179 server = {
180 http_addr = "";
181 http_port = ports.grafana;
182 };
183 security = {
184 admin_password = "$__file{${grafanaVars.admin-password.path}}";
185 secret_key = "$__file{${grafanaVars.secret-key.path}}";
186 };
187 "auth.anonymous" = {
188 enabled = true;
189 org_role = "Viewer";
190 };
191 analytics.reporting_enabled = false;
192 };
193 provision = {
194 enable = true;
195 datasources.settings.datasources = [
196 {
197 name = "VictoriaMetrics";
198 uid = "victoriametrics";
199 type = "victoriametrics-metrics-datasource";
200 url = "http://127.0.0.1:${toString ports.victoriametrics}";
201 isDefault = true;
202 }
203 {
204 name = "VictoriaLogs";
205 uid = "victorialogs";
206 type = "victoriametrics-logs-datasource";
207 url = "http://127.0.0.1:${toString ports.victorialogs}";
208 }
209 ];
210 dashboards.settings.providers = [
211 {
212 name = "monitoring";
213 options.path = ./dashboards;
214 }
215 ];
216 };
217 };
218
219 clan.core.vars.generators.monitoring-grafana = {
220 files.admin-password.owner = "grafana";
221 files.secret-key.owner = "grafana";
222 runtimeInputs = [
223 pkgs.coreutils
224 pkgs.openssl
225 ];
226 script = ''
227 openssl rand -hex 16 | tr -d '\n' > "$out/admin-password"
228 openssl rand -hex 32 | tr -d '\n' > "$out/secret-key"
229 '';
230 };
231
232 networking.firewall.interfaces.retiolum = {
233 allowedTCPPorts = with ports; [
234 victoriametrics
235 victorialogs
236 vmalert
237 alertmanager
238 alertmanager-cluster
239 gatus
240 grafana
241 ];
242 allowedUDPPorts = [ ports.alertmanager-cluster ];
243 };
244}