This repository has no description
5.8 kB
199 lines
1{
2 pkgs,
3 lib,
4 config,
5 ...
6}:
7# To use this module you also need to allow port 9273 either on the internet or on a vpn interface
8# i.e. networking.firewall.interfaces."vpn0".allowedTCPPorts = [ 9273 ];
9# Example prometheus alert rules:
10# - https://github.com/Mic92/dotfiles/blob/master/nixos/eva/modules/prometheus/alert-rules.nix
11let
12 isVM = lib.any (
13 mod: mod == "xen-blkfront" || mod == "virtio_console"
14 ) config.boot.initrd.kernelModules;
15 # potentially wrong if the nvme is not used at boot...
16 hasNvme = lib.any (m: m == "nvme") config.boot.initrd.availableKernelModules;
17
18 ipv6DadCheck = pkgs.writeShellScript "ipv6-dad-check" ''
19 ${pkgs.iproute2}/bin/ip --json addr | \
20 ${pkgs.jq}/bin/jq -r 'map(.addr_info) | flatten(1) | map(select(.dadfailed == true)) | map(.local) | @text "ipv6_dad_failures count=\(length)i"'
21 '';
22
23 zfsChecks =
24 lib.optional (lib.any (fs: fs == "zfs") (lib.attrNames config.boot.supportedFilesystems))
25 (
26 pkgs.writeScript "zpool-health" ''
27 #!${pkgs.gawk}/bin/awk -f
28 BEGIN {
29 while ("${pkgs.zfs}/bin/zpool status" | getline) {
30 if ($1 ~ /pool:/) { printf "zpool_status,name=%s ", $2 }
31 if ($1 ~ /state:/) { printf " state=\"%s\",", $2 }
32 if ($1 ~ /errors:/) {
33 if (index($2, "No")) printf "errors=0i\n"; else printf "errors=%di\n", $2
34 }
35 }
36 }
37 ''
38 );
39
40 nfsChecks =
41 let
42 collectHosts =
43 shares: fs:
44 if
45 builtins.elem fs.fsType [
46 "nfs"
47 "nfs3"
48 "nfs4"
49 ]
50 then
51 shares
52 // (
53 let
54 # also match ipv6 addresses
55 group = builtins.match "\\[?([^\]]+)]?:([^:]+)$" fs.device;
56 host = builtins.head group;
57 path = builtins.elemAt group 1;
58 in
59 {
60 ${host} = (shares.${host} or [ ]) ++ [ path ];
61 }
62 )
63 else
64 shares;
65 nfsHosts = lib.foldl collectHosts { } (builtins.attrValues config.fileSystems);
66 in
67 lib.mapAttrsToList (
68 host: args:
69 (pkgs.writeScript "nfs-health" ''
70 #!${pkgs.gawk}/bin/awk -f
71 BEGIN {
72 for (i = 2; i < ARGC; i++) {
73 mounts[ARGV[i]] = 1
74 }
75 while ("${pkgs.nfs-utils}/bin/showmount -e " ARGV[1] | getline) {
76 if (NR == 1) { continue }
77 if (mounts[$1] == 1) {
78 printf "nfs_export,host=%s,path=%s present=1\n", ARGV[1], $1
79 }
80 delete mounts[$1]
81 }
82 for (mount in mounts) {
83 printf "nfs_export,host=%s,path=%s present=0\n", ARGV[1], $1
84 }
85 }
86 '')
87 + " ${host} ${builtins.concatStringsSep " " args}"
88 ) nfsHosts;
89
90in
91{
92
93 systemd.services.telegraf.path = lib.optional (!isVM && hasNvme) pkgs.nvme-cli;
94
95 services.telegraf = {
96 enable = true;
97 extraConfig = {
98 agent.interval = "60s";
99 inputs =
100 {
101 prometheus.urls = lib.mkIf config.services.promtail.enable [
102 # default promtail port
103 "http://localhost:9080/metrics"
104 ];
105 prometheus.metric_version = 2;
106 kernel_vmstat = { };
107 nginx.urls = lib.mkIf config.services.nginx.statusPage [
108 "http://localhost/nginx_status"
109 ];
110 smart = lib.mkIf (!isVM) {
111 path_smartctl = pkgs.writeShellScript "smartctl" ''
112 exec /run/wrappers/bin/sudo ${pkgs.smartmontools}/bin/smartctl "$@"
113 '';
114 };
115 system = { };
116 mem = { };
117 file =
118 [
119 {
120 data_format = "influx";
121 file_tag = "name";
122 files = [ "/var/log/telegraf/*" ];
123 }
124 ]
125 ++ lib.optional (lib.any (fs: fs == "ext4") (lib.attrNames config.boot.supportedFilesystems)) {
126 name_override = "ext4_errors";
127 files = [ "/sys/fs/ext4/*/errors_count" ];
128 data_format = "value";
129 };
130 exec = [
131 {
132 ## Commands array
133 commands = [ ipv6DadCheck ] ++ zfsChecks ++ nfsChecks;
134 data_format = "influx";
135 }
136 ];
137 systemd_units = { };
138 swap = { };
139 disk.tagdrop = {
140 fstype = [
141 "tmpfs"
142 "ramfs"
143 "devtmpfs"
144 "devfs"
145 "iso9660"
146 "overlay"
147 "aufs"
148 "squashfs"
149 ];
150 device = [
151 "rpc_pipefs"
152 "lxcfs"
153 "nsfs"
154 "borgfs"
155 ];
156 };
157 diskio = { };
158 zfs = {
159 poolMetrics = true;
160 };
161 }
162 // lib.optionalAttrs
163 (
164 if lib.versionAtLeast (lib.versions.majorMinor lib.version) "23.11" then
165 config.boot.swraid.enable
166 else
167 config.boot.initrd.services.swraid.enable
168 )
169 {
170 mdstat = { };
171 };
172 outputs.prometheus_client = {
173 listen = ":9273";
174 metric_version = 2;
175 };
176 };
177 };
178 security.sudo.extraRules = lib.mkIf (!isVM) [
179 {
180 users = [ "telegraf" ];
181 commands = [
182 {
183 command = "${pkgs.smartmontools}/bin/smartctl";
184 options = [ "NOPASSWD" ];
185 }
186 ];
187 }
188 ];
189 networking.firewall.allowedTCPPorts = [ 9273 ];
190
191 # avoid logging sudo use
192 security.sudo.configFile = ''
193 Defaults:telegraf !syslog,!pam_session
194 '';
195 # create dummy file to avoid telegraf errors
196 systemd.tmpfiles.rules = [
197 "f /var/log/telegraf/dummy 0444 root root - -"
198 ];
199}