This repository has no description
12 kB
349 lines
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7let
8 cfg = config.krebs.sync-containers3;
9in
10{
11 options.krebs.sync-containers3 = {
12 inContainer = {
13 enable = lib.mkEnableOption "container config for syncing";
14 pubkey = lib.mkOption {
15 type = lib.types.str;
16 };
17 };
18 containers = lib.mkOption {
19 default = { };
20 type = lib.types.attrsOf (
21 lib.types.submodule (
22 { config, ... }:
23 {
24 options = {
25 name = lib.mkOption {
26 type = lib.types.str;
27 default = config._module.args.name;
28 };
29 sshKey = lib.mkOption {
30 type = lib.types.str;
31 };
32 luksKey = lib.mkOption {
33 type = lib.types.str;
34 default = config.sshKey;
35 };
36 ephemeral = lib.mkOption {
37 type = lib.types.bool;
38 default = false;
39 };
40 runContainer = lib.mkOption {
41 type = lib.types.bool;
42 default = true;
43 };
44 startCommand = lib.mkOption {
45 type = lib.types.str;
46 default = ''
47 set -efu
48 mkdir -p /var/state/var_src
49 ln -Tfrs /var/state/var_src /var/src
50 if test -e /var/src/nixos-config; then
51 /run/current-system/sw/bin/nixos-rebuild -I /var/src switch || :
52 fi
53 '';
54 };
55 hostname = lib.mkOption {
56 type = lib.types.str;
57 description = ''
58 hostname of the container,
59 this is continuously checked by ping and the container is restarted if unreachable
60 '';
61 default = config.name;
62 };
63 };
64 }
65 )
66 );
67 };
68 };
69 config = lib.mkMerge [
70 (lib.mkIf (cfg.containers != { }) {
71
72 containers = lib.mapAttrs' (
73 _n: ctr:
74 lib.nameValuePair ctr.name {
75 config = {
76 environment.systemPackages = [
77 pkgs.dhcpcd
78 pkgs.git
79 pkgs.jq
80 ];
81 networking.useDHCP = lib.mkForce true;
82 networking.useHostResolvConf = false;
83 services.resolved.enable = true;
84 systemd.services.autoswitch = {
85 environment = {
86 NIX_REMOTE = "daemon";
87 };
88 wantedBy = [ "multi-user.target" ];
89 serviceConfig = {
90 ExecStart = pkgs.writers.writeDash "autoswitch" ctr.startCommand;
91 # keep retrying transient failures (e.g. a GitHub 5xx while
92 # resolving the flake at boot) so we self-heal instead of
93 # staying silently stale. RestartSec is well above the default
94 # StartLimitIntervalSec, so the restart rate limiter never trips
95 # and we retry until it succeeds.
96 Restart = "on-failure";
97 RestartSec = "120s";
98 };
99 unitConfig.X-StopOnRemoval = false;
100 };
101 # get rid of stateVersion not set warning;
102 system.stateVersion = config.system.nixos.release;
103 };
104 autoStart = false;
105 enableTun = true;
106 ephemeral = ctr.ephemeral;
107 privateNetwork = true;
108 hostBridge = "ctr0";
109 bindMounts = {
110 "/var/lib/self/disk" = {
111 hostPath = "/var/lib/sync-containers3/${ctr.name}/disk";
112 isReadOnly = false;
113 };
114 "/var/state" = {
115 hostPath = "/var/lib/sync-containers3/${ctr.name}/state";
116 isReadOnly = false;
117 };
118 };
119 }
120 ) (lib.filterAttrs (_: ctr: ctr.runContainer) cfg.containers);
121
122 systemd.services = lib.foldr lib.recursiveUpdate { } (
123 lib.flatten (
124 map (ctr: [
125 {
126 "${ctr.name}_syncer" = {
127 path = with pkgs; [
128 coreutils
129 inetutils
130 rsync
131 openssh
132 systemd
133 util-linux
134 ];
135 startAt = "*:0/1";
136 serviceConfig = {
137 User = "${ctr.name}_container";
138 LoadCredential = [
139 "ssh_key:${ctr.sshKey}"
140 ];
141 ExecCondition = pkgs.writers.writeDash "${ctr.name}_checker" ''
142 set -efu
143 ! systemctl is-active --quiet container@${ctr.name}.service
144 '';
145 ExecStart = pkgs.writers.writeDash "${ctr.name}_syncer" ''
146 set -efux
147 flock -n "$HOME"/sync.lock ${pkgs.writers.writeDash "${ctr.name}-sync" ''
148 set -efux
149 if ping -c 1 ${ctr.hostname}; then
150 nice --adjustment=30 rsync -a -e "ssh -i $CREDENTIALS_DIRECTORY/ssh_key" --timeout=30 --inplace --sparse container_sync@${ctr.hostname}:disk "$HOME"/disk.rsync
151 touch "$HOME"/incomplete
152 nice --adjustment=30 rsync --inplace "$HOME"/disk.rsync "$HOME"/disk
153 rm -f "$HOME"/incomplete
154 fi
155 ''}
156 '';
157 };
158 };
159 }
160 {
161 "${ctr.name}_watcher" = lib.mkIf ctr.runContainer {
162 path = with pkgs; [
163 coreutils
164 inetutils
165 cryptsetup
166 mount
167 util-linux
168 retry
169 ];
170 serviceConfig = {
171 ExecStart = pkgs.writers.writeDash "${ctr.name}_watcher" ''
172 set -efux
173 while sleep 5; do
174 if ! $(retry -t 10 -d 10 -- ping -q -c 1 ${ctr.hostname} > /dev/null); then
175 echo 'container seems dead, killing'
176 break
177 fi
178 done
179 /run/current-system/sw/bin/nixos-container stop ${ctr.name} || :
180 umount /var/lib/sync-containers3/${ctr.name}/state || :
181 cryptsetup luksClose ${ctr.name} || :
182 '';
183 };
184 };
185 }
186 {
187 "${ctr.name}_scheduler" = lib.mkIf ctr.runContainer {
188 wantedBy = [ "multi-user.target" ];
189 path = with pkgs; [
190 coreutils
191 inetutils
192 cryptsetup
193 mount
194 util-linux
195 systemd
196 retry
197 ];
198 serviceConfig = {
199 Restart = "always";
200 RestartSec = "30s";
201 ExecStart = pkgs.writers.writeBash "${ctr.name}_scheduler" ''
202 set -efu
203 cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name} || :
204 mkdir -p /var/lib/sync-containers3/${ctr.name}/state
205 mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state
206 /run/current-system/sw/bin/nixos-container start ${ctr.name}
207 # wait for system to become reachable for the first time
208 systemctl start ${ctr.name}_watcher.service
209 retry -t 10 -d 10 -- ping -q -c 1 ${ctr.hostname} > /dev/null
210 while systemctl is-active container@${ctr.name}.service >/dev/null && ping -q -c 3 ${ctr.hostname} >/dev/null; do
211 sleep 10
212 done
213 '';
214 };
215 };
216 }
217 {
218 "container@${ctr.name}" = lib.mkIf ctr.runContainer {
219 serviceConfig = {
220 ExecStop = pkgs.writers.writeDash "remove_interface" ''
221 ${pkgs.iproute2}/bin/ip link del vb-${ctr.name}
222 '';
223 ExecStartPost = [
224 (pkgs.writers.writeDash "bind-to-bridge" ''
225 ${pkgs.iproute2}/bin/ip link set "vb-$INSTANCE" master ctr0
226 '')
227 ];
228 };
229 };
230 }
231 ]) (lib.attrValues cfg.containers)
232 )
233 );
234
235 systemd.timers = lib.mapAttrs' (
236 _n: ctr:
237 lib.nameValuePair "${ctr.name}_syncer" {
238 timerConfig = {
239 RandomizedDelaySec = 100;
240 };
241 }
242 ) cfg.containers;
243
244 users.groups = lib.mapAttrs' (
245 _: ctr:
246 lib.nameValuePair "${ctr.name}_container" {
247 }
248 ) cfg.containers;
249 users.users = lib.mapAttrs' (
250 _: ctr:
251 lib.nameValuePair "${ctr.name}_container" ({
252 group = "${ctr.name}_container";
253 isNormalUser = true;
254 home = "/var/lib/sync-containers3/${ctr.name}";
255 createHome = true;
256 homeMode = "705";
257 })
258 ) cfg.containers;
259
260 environment.systemPackages = lib.mapAttrsToList (
261 _: ctr:
262 (pkgs.writers.writeDashBin "${ctr.name}_init" ''
263 set -efux
264 export PATH=${
265 lib.makeBinPath [
266 pkgs.coreutils
267 pkgs.cryptsetup
268 pkgs.libxfs.bin
269 ]
270 }:$PATH
271 truncate -s 5G /var/lib/sync-containers3/${ctr.name}/disk
272 cryptsetup luksFormat /var/lib/sync-containers3/${ctr.name}/disk ${ctr.luksKey}
273 cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name}
274 mkfs.xfs /dev/mapper/${ctr.name}
275 mkdir -p /var/lib/sync-containers3/${ctr.name}/state
276 mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state
277 /run/current-system/sw/bin/nixos-container start ${ctr.name}
278 /run/current-system/sw/bin/nixos-container run ${ctr.name} -- ${pkgs.writers.writeDash "init" ''
279 mkdir -p /var/state
280 ''}
281 '')
282 ) cfg.containers;
283 })
284 (lib.mkIf (cfg.containers != { }) {
285 # networking
286
287 boot.kernel.sysctl."net.ipv4.ip_forward" = lib.mkForce 1;
288 systemd.network.networks.ctr0 = {
289 name = "ctr0";
290 address = [
291 "10.233.0.1/24"
292 ];
293 networkConfig = {
294 ConfigureWithoutCarrier = true;
295 DHCPServer = "yes";
296 };
297 dhcpServerConfig = {
298 DNS = "9.9.9.9";
299 };
300 };
301 systemd.network.netdevs.ctr0.netdevConfig = {
302 Kind = "bridge";
303 Name = "ctr0";
304 };
305 networking.networkmanager.unmanaged = [ "ctr0" ];
306 networking.firewall.trustedInterfaces = [ "ctr0" ];
307 networking.firewall.extraForwardRules = ''
308 iifname "ctr0" accept
309 oifname "ctr0" accept
310 '';
311 networking.nftables.tables.sync-containers3 = {
312 family = "ip";
313 content = ''
314 chain postrouting {
315 type nat hook postrouting priority srcnat; policy accept;
316 ip saddr 10.233.0.0/24 masquerade
317 }
318 '';
319 };
320 })
321 (lib.mkIf cfg.inContainer.enable {
322 users.groups.container_sync = { };
323 users.users.container_sync = {
324 group = "container_sync";
325 isNormalUser = true;
326 home = "/var/lib/self";
327 createHome = true;
328 openssh.authorizedKeys.keys = [
329 cfg.inContainer.pubkey
330 ];
331 };
332
333 networking.useHostResolvConf = false;
334 networking.useNetworkd = true;
335 services.resolved = {
336 enable = true;
337 settings.Resolve.Domains = [ "~." ];
338 };
339 systemd.network = {
340 enable = true;
341 networks.eth0 = {
342 matchConfig.Name = "eth0";
343 DHCP = "yes";
344 dhcpV4Config.UseDNS = true;
345 };
346 };
347 })
348 ];
349}