This repository has no description
12 kB
349 lines
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8 cfg = config.krebs.sync-containers3;
9in
10{
11 options.krebs.sync-containers3 = {
12 inContainer = {
13 enable = lib.mkEnableOption "container config for syncing";
14 pubkey = lib.mkOption {
15 type = lib.types.str;
16 };
17 };
18 containers = lib.mkOption {
19 default = { };
20 type = lib.types.attrsOf (
21 lib.types.submodule (
22 { config, ... }:
23 {
24 options = {
25 name = lib.mkOption {
26 type = lib.types.str;
27 default = config._module.args.name;
28 };
29 sshKey = lib.mkOption {
30 type = lib.types.str;
31 };
32 luksKey = lib.mkOption {
33 type = lib.types.str;
34 default = config.sshKey;
35 };
36 ephemeral = lib.mkOption {
37 type = lib.types.bool;
38 default = false;
39 };
40 runContainer = lib.mkOption {
41 type = lib.types.bool;
42 default = true;
43 };
44 startCommand = lib.mkOption {
45 type = lib.types.str;
46 default = ''
47 set -efu
48 mkdir -p /var/state/var_src
49 ln -Tfrs /var/state/var_src /var/src
50 if test -e /var/src/nixos-config; then
51 /run/current-system/sw/bin/nixos-rebuild -I /var/src switch || :
52 fi
53 '';
54 };
55 hostname = lib.mkOption {
56 type = lib.types.str;
57 description = ''
58 hostname of the container,
59 this is continuously checked by ping and the container is restarted if unreachable
60 '';
61 default = config.name;
62 };
63 };
64 }
65 )
66 );
67 };
68 };
69 config = lib.mkMerge [
70 (lib.mkIf (cfg.containers != { }) {
71
72 containers = lib.mapAttrs' (
73 _n: ctr:
74 lib.nameValuePair ctr.name {
75 config = {
76 environment.systemPackages = [
77 pkgs.dhcpcd
78 pkgs.git
79 pkgs.jq
80 ];
81 networking.useDHCP = lib.mkForce true;
82 networking.useHostResolvConf = false;
83 services.resolved.enable = true;
84 systemd.services.autoswitch = {
85 environment = {
86 NIX_REMOTE = "daemon";
87 };
88 wantedBy = [ "multi-user.target" ];
89 serviceConfig.ExecStart = pkgs.writers.writeDash "autoswitch" ctr.startCommand;
90 unitConfig.X-StopOnRemoval = false;
91 };
92 # get rid of stateVersion not set warning;
93 system.stateVersion = config.system.nixos.release;
94 };
95 autoStart = false;
96 enableTun = true;
97 ephemeral = ctr.ephemeral;
98 privateNetwork = true;
99 hostBridge = "ctr0";
100 bindMounts = {
101 "/var/lib/self/disk" = {
102 hostPath = "/var/lib/sync-containers3/${ctr.name}/disk";
103 isReadOnly = false;
104 };
105 "/var/state" = {
106 hostPath = "/var/lib/sync-containers3/${ctr.name}/state";
107 isReadOnly = false;
108 };
109 };
110 }
111 ) (lib.filterAttrs (_: ctr: ctr.runContainer) cfg.containers);
112
113 systemd.services = lib.foldr lib.recursiveUpdate { } (
114 lib.flatten (
115 map (ctr: [
116 {
117 "${ctr.name}_syncer" = {
118 path = with pkgs; [
119 coreutils
120 inetutils
121 rsync
122 openssh
123 systemd
124 util-linux
125 ];
126 startAt = "*:0/1";
127 serviceConfig = {
128 User = "${ctr.name}_container";
129 LoadCredential = [
130 "ssh_key:${ctr.sshKey}"
131 ];
132 ExecCondition = pkgs.writers.writeDash "${ctr.name}_checker" ''
133 set -efu
134 ! systemctl is-active --quiet container@${ctr.name}.service
135 '';
136 ExecStart = pkgs.writers.writeDash "${ctr.name}_syncer" ''
137 set -efux
138 flock -n "$HOME"/sync.lock ${pkgs.writers.writeDash "${ctr.name}-sync" ''
139 set -efux
140 if ping -c 1 ${ctr.hostname}; then
141 nice --adjustment=30 rsync -a -e "ssh -i $CREDENTIALS_DIRECTORY/ssh_key" --timeout=30 --inplace --sparse container_sync@${ctr.hostname}:disk "$HOME"/disk.rsync
142 touch "$HOME"/incomplete
143 nice --adjustment=30 rsync --inplace "$HOME"/disk.rsync "$HOME"/disk
144 rm -f "$HOME"/incomplete
145 fi
146 ''}
147 '';
148 };
149 };
150 }
151 {
152 "${ctr.name}_watcher" = lib.mkIf ctr.runContainer {
153 path = with pkgs; [
154 coreutils
155 inetutils
156 cryptsetup
157 mount
158 util-linux
159 retry
160 ];
161 serviceConfig = {
162 ExecStart = pkgs.writers.writeDash "${ctr.name}_watcher" ''
163 set -efux
164 while sleep 5; do
165 if ! $(retry -t 10 -d 10 -- ping -q -c 1 ${ctr.hostname} > /dev/null); then
166 echo 'container seems dead, killing'
167 break
168 fi
169 done
170 /run/current-system/sw/bin/nixos-container stop ${ctr.name} || :
171 umount /var/lib/sync-containers3/${ctr.name}/state || :
172 cryptsetup luksClose ${ctr.name} || :
173 '';
174 };
175 };
176 }
177 {
178 "${ctr.name}_scheduler" = lib.mkIf ctr.runContainer {
179 wantedBy = [ "multi-user.target" ];
180 path = with pkgs; [
181 coreutils
182 inetutils
183 cryptsetup
184 mount
185 util-linux
186 systemd
187 retry
188 ];
189 serviceConfig = {
190 Restart = "always";
191 RestartSec = "30s";
192 ExecStart = pkgs.writers.writeBash "${ctr.name}_scheduler" ''
193 set -efu
194 cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name} || :
195 mkdir -p /var/lib/sync-containers3/${ctr.name}/state
196 mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state
197 /run/current-system/sw/bin/nixos-container start ${ctr.name}
198 # wait for system to become reachable for the first time
199 systemctl start ${ctr.name}_watcher.service
200 retry -t 10 -d 10 -- ping -q -c 1 ${ctr.hostname} > /dev/null
201 while systemctl is-active container@${ctr.name}.service >/dev/null && ping -q -c 3 ${ctr.hostname} >/dev/null; do
202 sleep 10
203 done
204 '';
205 };
206 };
207 }
208 {
209 "container@${ctr.name}" = lib.mkIf ctr.runContainer {
210 serviceConfig = {
211 ExecStop = pkgs.writers.writeDash "remove_interface" ''
212 ${pkgs.iproute2}/bin/ip link del vb-${ctr.name}
213 '';
214 ExecStartPost = [
215 (pkgs.writers.writeDash "bind-to-bridge" ''
216 ${pkgs.iproute2}/bin/ip link set "vb-$INSTANCE" master ctr0
217 '')
218 ];
219 };
220 };
221 }
222 ]) (lib.attrValues cfg.containers)
223 )
224 );
225
226 systemd.timers = lib.mapAttrs' (
227 _n: ctr:
228 lib.nameValuePair "${ctr.name}_syncer" {
229 timerConfig = {
230 RandomizedDelaySec = 100;
231 };
232 }
233 ) cfg.containers;
234
235 users.groups = lib.mapAttrs' (
236 _: ctr:
237 lib.nameValuePair "${ctr.name}_container" {
238 }
239 ) cfg.containers;
240 users.users = lib.mapAttrs' (
241 _: ctr:
242 lib.nameValuePair "${ctr.name}_container" ({
243 group = "${ctr.name}_container";
244 isNormalUser = true;
245 home = "/var/lib/sync-containers3/${ctr.name}";
246 createHome = true;
247 homeMode = "705";
248 })
249 ) cfg.containers;
250
251 environment.systemPackages = lib.mapAttrsToList (
252 _: ctr:
253 (pkgs.writers.writeDashBin "${ctr.name}_init" ''
254 set -efux
255 export PATH=${
256 lib.makeBinPath [
257 pkgs.coreutils
258 pkgs.cryptsetup
259 pkgs.libxfs.bin
260 ]
261 }:$PATH
262 truncate -s 5G /var/lib/sync-containers3/${ctr.name}/disk
263 cryptsetup luksFormat /var/lib/sync-containers3/${ctr.name}/disk ${ctr.luksKey}
264 cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name}
265 mkfs.xfs /dev/mapper/${ctr.name}
266 mkdir -p /var/lib/sync-containers3/${ctr.name}/state
267 mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state
268 /run/current-system/sw/bin/nixos-container start ${ctr.name}
269 /run/current-system/sw/bin/nixos-container run ${ctr.name} -- ${pkgs.writeDash "init" ''
270 mkdir -p /var/state
271 ''}
272 '')
273 ) cfg.containers;
274 })
275 (lib.mkIf (cfg.containers != { }) {
276 # networking
277
278 boot.kernel.sysctl."net.ipv4.ip_forward" = lib.mkForce 1;
279 systemd.network.networks.ctr0 = {
280 name = "ctr0";
281 address = [
282 "10.233.0.1/24"
283 ];
284 networkConfig = {
285 ConfigureWithoutCarrier = true;
286 DHCPServer = "yes";
287 };
288 dhcpServerConfig = {
289 DNS = "9.9.9.9";
290 };
291 };
292 systemd.network.netdevs.ctr0.netdevConfig = {
293 Kind = "bridge";
294 Name = "ctr0";
295 };
296 networking.networkmanager.unmanaged = [ "ctr0" ];
297 krebs.iptables.tables.filter.INPUT.rules = [
298 {
299 predicate = "-i ctr0";
300 target = "ACCEPT";
301 }
302 ];
303 krebs.iptables.tables.filter.FORWARD.rules = [
304 {
305 predicate = "-i ctr0";
306 target = "ACCEPT";
307 }
308 {
309 predicate = "-o ctr0";
310 target = "ACCEPT";
311 }
312 ];
313 krebs.iptables.tables.nat.POSTROUTING.rules = [
314 {
315 v6 = false;
316 predicate = "-s 10.233.0.0/24";
317 target = "MASQUERADE";
318 }
319 ];
320 })
321 (lib.mkIf cfg.inContainer.enable {
322 users.groups.container_sync = { };
323 users.users.container_sync = {
324 group = "container_sync";
325 isNormalUser = true;
326 home = "/var/lib/self";
327 createHome = true;
328 openssh.authorizedKeys.keys = [
329 cfg.inContainer.pubkey
330 ];
331 };
332
333 networking.useHostResolvConf = false;
334 networking.useNetworkd = true;
335 services.resolved = {
336 enable = true;
337 settings.Resolve.Domains = [ "~." ];
338 };
339 systemd.network = {
340 enable = true;
341 networks.eth0 = {
342 matchConfig.Name = "eth0";
343 DHCP = "yes";
344 dhcpV4Config.UseDNS = true;
345 };
346 };
347 })
348 ];
349}