This repository has no description
5.8 kB
159 lines
1{ config, lib, pkgs, self, ... }: let
2 cfg = config.lass.drbd;
3 slib = self.inputs.stockholm.lib;
4in {
5 options = {
6 lass.drbd = lib.mkOption {
7 default = {};
8 type = lib.types.attrsOf (lib.types.submodule ({ config, ... }: {
9 options = {
10 name = lib.mkOption {
11 type = lib.types.str;
12 default = config._module.args.name;
13 };
14 blockMinor = lib.mkOption {
15 type = lib.types.int;
16 default = lib.mod (slib.genid config.name) 16000; # TODO get max_id fron drbd
17 };
18 port = lib.mkOption {
19 type = lib.types.int;
20 default = 20000 + config.blockMinor;
21 };
22 peers = lib.mkOption {
23 type = lib.types.listOf slib.types.host;
24 };
25 disk = lib.mkOption {
26 type = lib.types.str;
27 default = "/dev/loop${toString config.blockMinor}";
28 };
29 drbdConfig = lib.mkOption {
30 type = lib.types.path;
31 internal = true;
32 default = pkgs.writeText "drbd-${config.name}.conf" ''
33 resource ${config.name} {
34 net {
35 protocol a;
36 ping-int 10;
37 csums-alg crc32c;
38 connect-int 3;
39 after-sb-0pri discard-older-primary;
40 after-sb-1pri discard-secondary;
41
42 # seems to be drbd-proxy premium feature
43 on-congestion pull-ahead;
44 congestion-fill 1G;
45 congestion-extents 500;
46
47 sndbuf-size 10M;
48 max-epoch-size 20000;
49 }
50 device minor ${toString config.blockMinor};
51 disk ${config.disk};
52 meta-disk internal;
53 ${slib.indent (lib.concatStrings (lib.imap1 (i: peer: /* shell */ ''
54 on ${peer.name} {
55 address ${peer.nets.retiolum.ip4.addr}:${toString config.port};
56 node-id ${toString i};
57 }
58 '') config.peers))}
59 connection-mesh {
60 hosts ${lib.concatMapStringsSep " " (peer: peer.name) config.peers};
61 }
62 }
63 '';
64 };
65 };
66 }));
67 };
68 };
69 config = lib.mkIf (cfg != {}) {
70 boot.extraModulePackages = [
71 (pkgs.linuxPackages.callPackage ../5pkgs/drbd9/default.nix {})
72 ];
73 boot.extraModprobeConfig = ''
74 options drbd usermode_helper=/run/current-system/sw/bin/drbdadm
75 '';
76 services.udev.packages = [ pkgs.drbd ];
77 boot.kernelModules = [ "drbd" ];
78
79 environment.systemPackages = [
80 pkgs.drbd
81 (pkgs.writers.writeDashBin "drbd-change-nodeid" ''
82 # https://linbit.com/drbd-user-guide/drbd-guide-9_0-en/#s-using-truck-based-replication
83 set -efux
84
85 if [ "$#" -ne 2 ]; then
86 echo '$1 needs to be drbd volume name'
87 echo '$2 needs to be new node id'
88 exit 1
89 fi
90
91
92 TMPDIR=$(mktemp -d)
93 trap 'rm -rf $TMPDIR' EXIT
94
95 V=$1
96 NODE_TO=$2
97 META_DATA_LOCATION=internal
98
99 ${pkgs.drbd}/bin/drbdadm -- --force dump-md $V > "$TMPDIR"/md_orig.txt
100 NODE_FROM=$(cat "$TMPDIR"/md_orig.txt | ${pkgs.gnused}/bin/sed -n 's/^node-id \(.*\);$/\1/p')
101 ${pkgs.gnused}/bin/sed -e "s/node-id $NODE_FROM/node-id $NODE_TO/" \
102 -e "s/^peer.$NODE_FROM. /peer-NEW /" \
103 -e "s/^peer.$NODE_TO. /peer[$NODE_FROM] /" \
104 -e "s/^peer-NEW /peer[$NODE_TO] /" \
105 < "$TMPDIR"/md_orig.txt > "$TMPDIR"/md.txt
106
107 drbdmeta --force $(drbdadm sh-minor $V) v09 $(drbdadm sh-md-dev $V) $META_DATA_LOCATION restore-md "$TMPDIR"/md.txt
108 '')
109 ];
110
111 networking.firewall.allowedTCPPorts = map (device: device.port) (lib.attrValues cfg);
112 systemd.services = lib.mapAttrs' (_: device:
113 lib.nameValuePair "drbd-${device.name}" {
114 after = [ "systemd-udev.settle.service" "network.target" "retiolum.service" ];
115 wants = [ "systemd-udev.settle.service" ];
116 wantedBy = [ "multi-user.target" ];
117 serviceConfig = {
118 RemainAfterExit = true;
119 ExecStart = pkgs.writers.writeDash "start-drbd-${device.name}" ''
120 set -efux
121 mkdir -p /var/lib/sync-containers2
122 ${lib.optionalString (device.disk == "/dev/loop${toString device.blockMinor}") ''
123 if ! test -e /var/lib/sync-containers2/${device.name}.disk; then
124 truncate -s 10G /var/lib/sync-containers2/${device.name}.disk
125 fi
126 if ! ${pkgs.util-linux}/bin/losetup /dev/loop${toString device.blockMinor}; then
127 ${pkgs.util-linux}/bin/losetup /dev/loop${toString device.blockMinor} /var/lib/sync-containers2/${device.name}.disk
128 fi
129 ''}
130 if ! ${pkgs.drbd}/bin/drbdadm adjust ${device.name}; then
131 ${pkgs.drbd}/bin/drbdadm down ${device.name}
132 ${pkgs.drbd}/bin/drbdadm create-md ${device.name}/0 --max-peers 31
133 ${pkgs.drbd}/bin/drbdadm up ${device.name}
134 fi
135 '';
136 ExecStop = pkgs.writers.writeDash "stop-drbd-${device.name}" ''
137 set -efux
138 ${pkgs.drbd}/bin/drbdadm -c ${device.drbdConfig} down ${device.name}
139 ${lib.optionalString (device.disk == "/dev/loop${toString device.blockMinor}") ''
140 ${pkgs.util-linux}/bin/losetup -d /dev/loop${toString device.blockMinor}
141 ''}
142 '';
143 };
144 }
145 ) cfg;
146
147
148 environment.etc."drbd.conf".text = ''
149 global {
150 usage-count yes;
151 }
152
153 ${lib.concatMapStrings (device: /* shell */ ''
154 include ${device.drbdConfig};
155 '') (lib.attrValues cfg)}
156 '';
157 };
158}
159