This repository has no description
5.4 kB
165 lines
1{ pkgs, lib, config, ... }:
2# To use this module you also need to allow port 9273 either on the internet or on a vpn interface
3# i.e. networking.firewall.interfaces."vpn0".allowedTCPPorts = [ 9273 ];
4# Example prometheus alert rules:
5# - https://github.com/Mic92/dotfiles/blob/master/nixos/eva/modules/prometheus/alert-rules.nix
6let
7 isVM = lib.any (mod: mod == "xen-blkfront" || mod == "virtio_console") config.boot.initrd.kernelModules;
8 # potentially wrong if the nvme is not used at boot...
9 hasNvme = lib.any (m: m == "nvme") config.boot.initrd.availableKernelModules;
10
11 ipv6DadCheck = pkgs.writeShellScript "ipv6-dad-check" ''
12 ${pkgs.iproute2}/bin/ip --json addr | \
13 ${pkgs.jq}/bin/jq -r 'map(.addr_info) | flatten(1) | map(select(.dadfailed == true)) | map(.local) | @text "ipv6_dad_failures count=\(length)i"'
14 '';
15
16 zfsChecks = lib.optional
17 (lib.any (fs: fs == "zfs") config.boot.supportedFilesystems)
18 (pkgs.writeScript "zpool-health" ''
19 #!${pkgs.gawk}/bin/awk -f
20 BEGIN {
21 while ("${pkgs.zfs}/bin/zpool status" | getline) {
22 if ($1 ~ /pool:/) { printf "zpool_status,name=%s ", $2 }
23 if ($1 ~ /state:/) { printf " state=\"%s\",", $2 }
24 if ($1 ~ /errors:/) {
25 if (index($2, "No")) printf "errors=0i\n"; else printf "errors=%di\n", $2
26 }
27 }
28 }
29 '');
30
31 nfsChecks =
32 let
33 collectHosts = shares: fs:
34 if builtins.elem fs.fsType [ "nfs" "nfs3" "nfs4" ]
35 then
36 shares
37 // (
38 let
39 # also match ipv6 addresses
40 group = builtins.match "\\[?([^\]]+)]?:([^:]+)$" fs.device;
41 host = builtins.head group;
42 path = builtins.elemAt group 1;
43 in
44 {
45 ${host} = (shares.${host} or [ ]) ++ [ path ];
46 }
47 )
48 else shares;
49 nfsHosts = lib.foldl collectHosts { } (builtins.attrValues config.fileSystems);
50 in
51 lib.mapAttrsToList
52 (
53 host: args:
54 (pkgs.writeScript "nfs-health" ''
55 #!${pkgs.gawk}/bin/awk -f
56 BEGIN {
57 for (i = 2; i < ARGC; i++) {
58 mounts[ARGV[i]] = 1
59 }
60 while ("${pkgs.nfs-utils}/bin/showmount -e " ARGV[1] | getline) {
61 if (NR == 1) { continue }
62 if (mounts[$1] == 1) {
63 printf "nfs_export,host=%s,path=%s present=1\n", ARGV[1], $1
64 }
65 delete mounts[$1]
66 }
67 for (mount in mounts) {
68 printf "nfs_export,host=%s,path=%s present=0\n", ARGV[1], $1
69 }
70 }
71 '')
72 + " ${host} ${builtins.concatStringsSep " " args}"
73 )
74 nfsHosts;
75
76in
77{
78
79 systemd.services.telegraf.path = lib.optional (!isVM && hasNvme) pkgs.nvme-cli;
80
81 services.telegraf = {
82 enable = true;
83 extraConfig = {
84 agent.interval = "60s";
85 inputs = {
86 prometheus.urls = lib.mkIf config.services.promtail.enable [
87 # default promtail port
88 "http://localhost:9080/metrics"
89 ];
90 prometheus.metric_version = 2;
91 kernel_vmstat = { };
92 nginx.urls = lib.mkIf config.services.nginx.statusPage [
93 "http://localhost/nginx_status"
94 ];
95 smart = lib.mkIf (!isVM) {
96 path_smartctl = pkgs.writeShellScript "smartctl" ''
97 exec /run/wrappers/bin/sudo ${pkgs.smartmontools}/bin/smartctl "$@"
98 '';
99 };
100 system = { };
101 mem = { };
102 file =
103 [
104 {
105 data_format = "influx";
106 file_tag = "name";
107 files = [ "/var/log/telegraf/*" ];
108 }
109 ]
110 ++ lib.optional (lib.any (fs: fs == "ext4") config.boot.supportedFilesystems) {
111 name_override = "ext4_errors";
112 files = [ "/sys/fs/ext4/*/errors_count" ];
113 data_format = "value";
114 };
115 exec = [
116 {
117 ## Commands array
118 commands =
119 [ ipv6DadCheck ]
120 ++ zfsChecks
121 ++ nfsChecks;
122 data_format = "influx";
123 }
124 ];
125 systemd_units = { };
126 swap = { };
127 disk.tagdrop = {
128 fstype = [ "tmpfs" "ramfs" "devtmpfs" "devfs" "iso9660" "overlay" "aufs" "squashfs" ];
129 device = [ "rpc_pipefs" "lxcfs" "nsfs" "borgfs" ];
130 };
131 diskio = { };
132 zfs = {
133 poolMetrics = true;
134 };
135 } // lib.optionalAttrs (if lib.versionAtLeast (lib.versions.majorMinor lib.version) "23.11" then config.boot.swraid.enable else config.boot.initrd.services.swraid.enable) {
136 mdstat = { };
137 };
138 outputs.prometheus_client = {
139 listen = ":9273";
140 metric_version = 2;
141 };
142 };
143 };
144 security.sudo.extraRules = lib.mkIf (!isVM) [
145 {
146 users = [ "telegraf" ];
147 commands = [
148 {
149 command = "${pkgs.smartmontools}/bin/smartctl";
150 options = [ "NOPASSWD" ];
151 }
152 ];
153 }
154 ];
155 networking.firewall.allowedTCPPorts = [ 9273 ];
156
157 # avoid logging sudo use
158 security.sudo.configFile = ''
159 Defaults:telegraf !syslog,!pam_session
160 '';
161 # create dummy file to avoid telegraf errors
162 systemd.tmpfiles.rules = [
163 "f /var/log/telegraf/dummy 0444 root root - -"
164 ];
165}