This repository has no description
3modules/sync-containers3: vendor from stockholm, remove consul dependency
author
lassulus
date
(Feb 6, 2026, 9:14 PM CET)
commit
3f4acf65
3f4acf65a648fa3944b5c9dd4c2e4886bb4a34bc
parent
f22254c5
f22254c518e519daad873503878df2b941937957
+350
-1
-1
2configs/default.nix
-1
2configs/default.nix
+1
3modules/default.nix
+1
3modules/default.nix
+349
3modules/sync-containers3.nix
+349
3modules/sync-containers3.nix
···
1
+
{
2
+
config,
3
+
lib,
4
+
pkgs,
5
+
...
6
+
}:
7
+
let
8
+
cfg = config.krebs.sync-containers3;
9
+
in
10
+
{
11
+
options.krebs.sync-containers3 = {
12
+
inContainer = {
13
+
enable = lib.mkEnableOption "container config for syncing";
14
+
pubkey = lib.mkOption {
15
+
type = lib.types.str;
16
+
};
17
+
};
18
+
containers = lib.mkOption {
19
+
default = { };
20
+
type = lib.types.attrsOf (
21
+
lib.types.submodule (
22
+
{ config, ... }:
23
+
{
24
+
options = {
25
+
name = lib.mkOption {
26
+
type = lib.types.str;
27
+
default = config._module.args.name;
28
+
};
29
+
sshKey = lib.mkOption {
30
+
type = lib.types.str;
31
+
};
32
+
luksKey = lib.mkOption {
33
+
type = lib.types.str;
34
+
default = config.sshKey;
35
+
};
36
+
ephemeral = lib.mkOption {
37
+
type = lib.types.bool;
38
+
default = false;
39
+
};
40
+
runContainer = lib.mkOption {
41
+
type = lib.types.bool;
42
+
default = true;
43
+
};
44
+
startCommand = lib.mkOption {
45
+
type = lib.types.str;
46
+
default = ''
47
+
set -efu
48
+
mkdir -p /var/state/var_src
49
+
ln -Tfrs /var/state/var_src /var/src
50
+
if test -e /var/src/nixos-config; then
51
+
/run/current-system/sw/bin/nixos-rebuild -I /var/src switch || :
52
+
fi
53
+
'';
54
+
};
55
+
hostname = lib.mkOption {
56
+
type = lib.types.str;
57
+
description = ''
58
+
hostname of the container,
59
+
this is continuously checked by ping and the container is restarted if unreachable
60
+
'';
61
+
default = config.name;
62
+
};
63
+
};
64
+
}
65
+
)
66
+
);
67
+
};
68
+
};
69
+
config = lib.mkMerge [
70
+
(lib.mkIf (cfg.containers != { }) {
71
+
72
+
containers = lib.mapAttrs' (
73
+
_n: ctr:
74
+
lib.nameValuePair ctr.name {
75
+
config = {
76
+
environment.systemPackages = [
77
+
pkgs.dhcpcd
78
+
pkgs.git
79
+
pkgs.jq
80
+
];
81
+
networking.useDHCP = lib.mkForce true;
82
+
networking.useHostResolvConf = false;
83
+
services.resolved.enable = true;
84
+
systemd.services.autoswitch = {
85
+
environment = {
86
+
NIX_REMOTE = "daemon";
87
+
};
88
+
wantedBy = [ "multi-user.target" ];
89
+
serviceConfig.ExecStart = pkgs.writers.writeDash "autoswitch" ctr.startCommand;
90
+
unitConfig.X-StopOnRemoval = false;
91
+
};
92
+
# get rid of stateVersion not set warning;
93
+
system.stateVersion = config.system.nixos.release;
94
+
};
95
+
autoStart = false;
96
+
enableTun = true;
97
+
ephemeral = ctr.ephemeral;
98
+
privateNetwork = true;
99
+
hostBridge = "ctr0";
100
+
bindMounts = {
101
+
"/var/lib/self/disk" = {
102
+
hostPath = "/var/lib/sync-containers3/${ctr.name}/disk";
103
+
isReadOnly = false;
104
+
};
105
+
"/var/state" = {
106
+
hostPath = "/var/lib/sync-containers3/${ctr.name}/state";
107
+
isReadOnly = false;
108
+
};
109
+
};
110
+
}
111
+
) (lib.filterAttrs (_: ctr: ctr.runContainer) cfg.containers);
112
+
113
+
systemd.services = lib.foldr lib.recursiveUpdate { } (
114
+
lib.flatten (
115
+
map (ctr: [
116
+
{
117
+
"${ctr.name}_syncer" = {
118
+
path = with pkgs; [
119
+
coreutils
120
+
inetutils
121
+
rsync
122
+
openssh
123
+
systemd
124
+
util-linux
125
+
];
126
+
startAt = "*:0/1";
127
+
serviceConfig = {
128
+
User = "${ctr.name}_container";
129
+
LoadCredential = [
130
+
"ssh_key:${ctr.sshKey}"
131
+
];
132
+
ExecCondition = pkgs.writers.writeDash "${ctr.name}_checker" ''
133
+
set -efu
134
+
! systemctl is-active --quiet container@${ctr.name}.service
135
+
'';
136
+
ExecStart = pkgs.writers.writeDash "${ctr.name}_syncer" ''
137
+
set -efux
138
+
flock -n "$HOME"/sync.lock ${pkgs.writers.writeDash "${ctr.name}-sync" ''
139
+
set -efux
140
+
if ping -c 1 ${ctr.hostname}; then
141
+
nice --adjustment=30 rsync -a -e "ssh -i $CREDENTIALS_DIRECTORY/ssh_key" --timeout=30 --inplace --sparse container_sync@${ctr.hostname}:disk "$HOME"/disk.rsync
142
+
touch "$HOME"/incomplete
143
+
nice --adjustment=30 rsync --inplace "$HOME"/disk.rsync "$HOME"/disk
144
+
rm -f "$HOME"/incomplete
145
+
fi
146
+
''}
147
+
'';
148
+
};
149
+
};
150
+
}
151
+
{
152
+
"${ctr.name}_watcher" = lib.mkIf ctr.runContainer {
153
+
path = with pkgs; [
154
+
coreutils
155
+
inetutils
156
+
cryptsetup
157
+
mount
158
+
util-linux
159
+
retry
160
+
];
161
+
serviceConfig = {
162
+
ExecStart = pkgs.writers.writeDash "${ctr.name}_watcher" ''
163
+
set -efux
164
+
while sleep 5; do
165
+
if ! $(retry -t 10 -d 10 -- ping -q -c 1 ${ctr.hostname} > /dev/null); then
166
+
echo 'container seems dead, killing'
167
+
break
168
+
fi
169
+
done
170
+
/run/current-system/sw/bin/nixos-container stop ${ctr.name} || :
171
+
umount /var/lib/sync-containers3/${ctr.name}/state || :
172
+
cryptsetup luksClose ${ctr.name} || :
173
+
'';
174
+
};
175
+
};
176
+
}
177
+
{
178
+
"${ctr.name}_scheduler" = lib.mkIf ctr.runContainer {
179
+
wantedBy = [ "multi-user.target" ];
180
+
path = with pkgs; [
181
+
coreutils
182
+
inetutils
183
+
cryptsetup
184
+
mount
185
+
util-linux
186
+
systemd
187
+
retry
188
+
];
189
+
serviceConfig = {
190
+
Restart = "always";
191
+
RestartSec = "30s";
192
+
ExecStart = pkgs.writers.writeBash "${ctr.name}_scheduler" ''
193
+
set -efu
194
+
cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name} || :
195
+
mkdir -p /var/lib/sync-containers3/${ctr.name}/state
196
+
mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state
197
+
/run/current-system/sw/bin/nixos-container start ${ctr.name}
198
+
# wait for system to become reachable for the first time
199
+
systemctl start ${ctr.name}_watcher.service
200
+
retry -t 10 -d 10 -- ping -q -c 1 ${ctr.hostname} > /dev/null
201
+
while systemctl is-active container@${ctr.name}.service >/dev/null && ping -q -c 3 ${ctr.hostname} >/dev/null; do
202
+
sleep 10
203
+
done
204
+
'';
205
+
};
206
+
};
207
+
}
208
+
{
209
+
"container@${ctr.name}" = lib.mkIf ctr.runContainer {
210
+
serviceConfig = {
211
+
ExecStop = pkgs.writers.writeDash "remove_interface" ''
212
+
${pkgs.iproute2}/bin/ip link del vb-${ctr.name}
213
+
'';
214
+
ExecStartPost = [
215
+
(pkgs.writers.writeDash "bind-to-bridge" ''
216
+
${pkgs.iproute2}/bin/ip link set "vb-$INSTANCE" master ctr0
217
+
'')
218
+
];
219
+
};
220
+
};
221
+
}
222
+
]) (lib.attrValues cfg.containers)
223
+
)
224
+
);
225
+
226
+
systemd.timers = lib.mapAttrs' (
227
+
_n: ctr:
228
+
lib.nameValuePair "${ctr.name}_syncer" {
229
+
timerConfig = {
230
+
RandomizedDelaySec = 100;
231
+
};
232
+
}
233
+
) cfg.containers;
234
+
235
+
users.groups = lib.mapAttrs' (
236
+
_: ctr:
237
+
lib.nameValuePair "${ctr.name}_container" {
238
+
}
239
+
) cfg.containers;
240
+
users.users = lib.mapAttrs' (
241
+
_: ctr:
242
+
lib.nameValuePair "${ctr.name}_container" ({
243
+
group = "${ctr.name}_container";
244
+
isNormalUser = true;
245
+
home = "/var/lib/sync-containers3/${ctr.name}";
246
+
createHome = true;
247
+
homeMode = "705";
248
+
})
249
+
) cfg.containers;
250
+
251
+
environment.systemPackages = lib.mapAttrsToList (
252
+
_: ctr:
253
+
(pkgs.writers.writeDashBin "${ctr.name}_init" ''
254
+
set -efux
255
+
export PATH=${
256
+
lib.makeBinPath [
257
+
pkgs.coreutils
258
+
pkgs.cryptsetup
259
+
pkgs.libxfs.bin
260
+
]
261
+
}:$PATH
262
+
truncate -s 5G /var/lib/sync-containers3/${ctr.name}/disk
263
+
cryptsetup luksFormat /var/lib/sync-containers3/${ctr.name}/disk ${ctr.luksKey}
264
+
cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name}
265
+
mkfs.xfs /dev/mapper/${ctr.name}
266
+
mkdir -p /var/lib/sync-containers3/${ctr.name}/state
267
+
mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state
268
+
/run/current-system/sw/bin/nixos-container start ${ctr.name}
269
+
/run/current-system/sw/bin/nixos-container run ${ctr.name} -- ${pkgs.writeDash "init" ''
270
+
mkdir -p /var/state
271
+
''}
272
+
'')
273
+
) cfg.containers;
274
+
})
275
+
(lib.mkIf (cfg.containers != { }) {
276
+
# networking
277
+
278
+
boot.kernel.sysctl."net.ipv4.ip_forward" = lib.mkForce 1;
279
+
systemd.network.networks.ctr0 = {
280
+
name = "ctr0";
281
+
address = [
282
+
"10.233.0.1/24"
283
+
];
284
+
networkConfig = {
285
+
ConfigureWithoutCarrier = true;
286
+
DHCPServer = "yes";
287
+
};
288
+
dhcpServerConfig = {
289
+
DNS = "9.9.9.9";
290
+
};
291
+
};
292
+
systemd.network.netdevs.ctr0.netdevConfig = {
293
+
Kind = "bridge";
294
+
Name = "ctr0";
295
+
};
296
+
networking.networkmanager.unmanaged = [ "ctr0" ];
297
+
krebs.iptables.tables.filter.INPUT.rules = [
298
+
{
299
+
predicate = "-i ctr0";
300
+
target = "ACCEPT";
301
+
}
302
+
];
303
+
krebs.iptables.tables.filter.FORWARD.rules = [
304
+
{
305
+
predicate = "-i ctr0";
306
+
target = "ACCEPT";
307
+
}
308
+
{
309
+
predicate = "-o ctr0";
310
+
target = "ACCEPT";
311
+
}
312
+
];
313
+
krebs.iptables.tables.nat.POSTROUTING.rules = [
314
+
{
315
+
v6 = false;
316
+
predicate = "-s 10.233.0.0/24";
317
+
target = "MASQUERADE";
318
+
}
319
+
];
320
+
})
321
+
(lib.mkIf cfg.inContainer.enable {
322
+
users.groups.container_sync = { };
323
+
users.users.container_sync = {
324
+
group = "container_sync";
325
+
isNormalUser = true;
326
+
home = "/var/lib/self";
327
+
createHome = true;
328
+
openssh.authorizedKeys.keys = [
329
+
cfg.inContainer.pubkey
330
+
];
331
+
};
332
+
333
+
networking.useHostResolvConf = false;
334
+
networking.useNetworkd = true;
335
+
services.resolved = {
336
+
enable = true;
337
+
settings.Resolve.Domains = [ "~." ];
338
+
};
339
+
systemd.network = {
340
+
enable = true;
341
+
networks.eth0 = {
342
+
matchConfig.Name = "eth0";
343
+
DHCP = "yes";
344
+
dhcpV4Config.UseDNS = true;
345
+
};
346
+
};
347
+
})
348
+
];
349
+
}