This repository has no description
6.4 kB
185 lines
1{
2 config,
3 lib,
4 pkgs,
5 self,
6 ...
7}:
8let
9 cfg = config.lass.drbd;
10 slib = self.inputs.stockholm.lib;
11in
12{
13 options = {
14 lass.drbd = lib.mkOption {
15 default = { };
16 type = lib.types.attrsOf (
17 lib.types.submodule (
18 { config, ... }:
19 {
20 options = {
21 name = lib.mkOption {
22 type = lib.types.str;
23 default = config._module.args.name;
24 };
25 blockMinor = lib.mkOption {
26 type = lib.types.int;
27 default = lib.mod (slib.genid config.name) 16000; # TODO get max_id fron drbd
28 };
29 port = lib.mkOption {
30 type = lib.types.int;
31 default = 20000 + config.blockMinor;
32 };
33 peers = lib.mkOption {
34 type = lib.types.listOf slib.types.host;
35 };
36 disk = lib.mkOption {
37 type = lib.types.str;
38 default = "/dev/loop${toString config.blockMinor}";
39 };
40 drbdConfig = lib.mkOption {
41 type = lib.types.path;
42 internal = true;
43 default = pkgs.writeText "drbd-${config.name}.conf" ''
44 resource ${config.name} {
45 net {
46 protocol a;
47 ping-int 10;
48 csums-alg crc32c;
49 connect-int 3;
50 after-sb-0pri discard-older-primary;
51 after-sb-1pri discard-secondary;
52
53 # seems to be drbd-proxy premium feature
54 on-congestion pull-ahead;
55 congestion-fill 1G;
56 congestion-extents 500;
57
58 sndbuf-size 10M;
59 max-epoch-size 20000;
60 }
61 device minor ${toString config.blockMinor};
62 disk ${config.disk};
63 meta-disk internal;
64 ${
65 slib.indent (
66 lib.concatStrings (
67 lib.imap1 (
68 i: peer: # shell
69 ''
70 on ${peer.name} {
71 address ${peer.nets.retiolum.ip4.addr}:${toString config.port};
72 node-id ${toString i};
73 }
74 '') config.peers
75 )
76 )
77 }
78 connection-mesh {
79 hosts ${lib.concatMapStringsSep " " (peer: peer.name) config.peers};
80 }
81 }
82 '';
83 };
84 };
85 }
86 )
87 );
88 };
89 };
90 config = lib.mkIf (cfg != { }) {
91 boot.extraModulePackages = [
92 (pkgs.linuxPackages.callPackage ../5pkgs/drbd9/default.nix { })
93 ];
94 boot.extraModprobeConfig = ''
95 options drbd usermode_helper=/run/current-system/sw/bin/drbdadm
96 '';
97 services.udev.packages = [ pkgs.drbd ];
98 boot.kernelModules = [ "drbd" ];
99
100 environment.systemPackages = [
101 pkgs.drbd
102 (pkgs.writers.writeDashBin "drbd-change-nodeid" ''
103 # https://linbit.com/drbd-user-guide/drbd-guide-9_0-en/#s-using-truck-based-replication
104 set -efux
105
106 if [ "$#" -ne 2 ]; then
107 echo '$1 needs to be drbd volume name'
108 echo '$2 needs to be new node id'
109 exit 1
110 fi
111
112
113 TMPDIR=$(mktemp -d)
114 trap 'rm -rf $TMPDIR' EXIT
115
116 V=$1
117 NODE_TO=$2
118 META_DATA_LOCATION=internal
119
120 ${pkgs.drbd}/bin/drbdadm -- --force dump-md $V > "$TMPDIR"/md_orig.txt
121 NODE_FROM=$(cat "$TMPDIR"/md_orig.txt | ${pkgs.gnused}/bin/sed -n 's/^node-id \(.*\);$/\1/p')
122 ${pkgs.gnused}/bin/sed -e "s/node-id $NODE_FROM/node-id $NODE_TO/" \
123 -e "s/^peer.$NODE_FROM. /peer-NEW /" \
124 -e "s/^peer.$NODE_TO. /peer[$NODE_FROM] /" \
125 -e "s/^peer-NEW /peer[$NODE_TO] /" \
126 < "$TMPDIR"/md_orig.txt > "$TMPDIR"/md.txt
127
128 drbdmeta --force $(drbdadm sh-minor $V) v09 $(drbdadm sh-md-dev $V) $META_DATA_LOCATION restore-md "$TMPDIR"/md.txt
129 '')
130 ];
131
132 networking.firewall.allowedTCPPorts = map (device: device.port) (lib.attrValues cfg);
133 systemd.services = lib.mapAttrs' (
134 _: device:
135 lib.nameValuePair "drbd-${device.name}" {
136 after = [
137 "systemd-udev.settle.service"
138 "network.target"
139 "retiolum.service"
140 ];
141 wants = [ "systemd-udev.settle.service" ];
142 wantedBy = [ "multi-user.target" ];
143 serviceConfig = {
144 RemainAfterExit = true;
145 ExecStart = pkgs.writers.writeDash "start-drbd-${device.name}" ''
146 set -efux
147 mkdir -p /var/lib/sync-containers2
148 ${lib.optionalString (device.disk == "/dev/loop${toString device.blockMinor}") ''
149 if ! test -e /var/lib/sync-containers2/${device.name}.disk; then
150 truncate -s 10G /var/lib/sync-containers2/${device.name}.disk
151 fi
152 if ! ${pkgs.util-linux}/bin/losetup /dev/loop${toString device.blockMinor}; then
153 ${pkgs.util-linux}/bin/losetup /dev/loop${toString device.blockMinor} /var/lib/sync-containers2/${device.name}.disk
154 fi
155 ''}
156 if ! ${pkgs.drbd}/bin/drbdadm adjust ${device.name}; then
157 ${pkgs.drbd}/bin/drbdadm down ${device.name}
158 ${pkgs.drbd}/bin/drbdadm create-md ${device.name}/0 --max-peers 31
159 ${pkgs.drbd}/bin/drbdadm up ${device.name}
160 fi
161 '';
162 ExecStop = pkgs.writers.writeDash "stop-drbd-${device.name}" ''
163 set -efux
164 ${pkgs.drbd}/bin/drbdadm -c ${device.drbdConfig} down ${device.name}
165 ${lib.optionalString (device.disk == "/dev/loop${toString device.blockMinor}") ''
166 ${pkgs.util-linux}/bin/losetup -d /dev/loop${toString device.blockMinor}
167 ''}
168 '';
169 };
170 }
171 ) cfg;
172
173 environment.etc."drbd.conf".text = ''
174 global {
175 usage-count yes;
176 }
177
178 ${lib.concatMapStrings (
179 device: # shell
180 ''
181 include ${device.drbdConfig};
182 '') (lib.attrValues cfg)}
183 '';
184 };
185}