forked from
lass.pds.lassul.us/superconfig
This repository has no description
modules.backup: per-service backup state, move neoprism paths into their services
lass.backup.state.<name> lets every service declare its folders, excludes,
pre/post backup hooks and the paths its pre hook writes to next to its
definition. The borg job (still `hetzner`, so archive names and pruning
continue) and the borgbackup ssh key generator exist only on machines that
set lass.backup.repo; neoprism's backup.nix now only points to the storage
box.
Pre hooks run inside the borg unit as before; each runs on its own with
pipefail, all of them run and the job fails if any did (a failing pg_dump
used to go unnoticed behind the gzip pipe). Post hooks run in ExecStopPost
as root, also after a failed backup.
The four bridge dumps are split from the old loop into the bridges' files.
The dump subdirs are created by the hooks instead of tmpfiles. hedgedoc's
uploads directory is backed up now.
The imgur items (hedgedoc image uploads) are backed up now too.
author
Hermes
committer
lassulus
date
(Oct 19, 2026, 7:46 AM CEST)
commit
7c67d7e6
7c67d7e6d8449278dc697532a7c2879138fcc5e3
parent
adf2387a
adf2387ad35fd8b7d16b1be3b2688ceebe4fec7b
+284
-117
+7
configs/mailserver.nix
+7
configs/mailserver.nix
···
99
99
imapMemoryLimit = 1024;
100
100
};
101
101
102
+
lass.backup.state.mailserver.folders = [
103
+
"/var/vmail"
104
+
"/var/dkim"
105
+
# global sieve dir from before stateVersion 5
106
+
"/var/sieve"
107
+
];
108
+
102
109
# nixos-mailserver mkForces virtualMail without homeMode; match its priority
103
110
users.users.virtualMail = lib.mkOverride 50 { homeMode = "2770"; };
104
111
+2
configs/radicale.nix
+2
configs/radicale.nix
+5
configs/timeful.nix
+5
configs/timeful.nix
···
21
21
package = pkgs.mongodb-ce;
22
22
};
23
23
24
+
lass.backup.state.timeful.preBackup = ''
25
+
${pkgs.mongodb-tools}/bin/mongodump --uri=mongodb://127.0.0.1:27017 \
26
+
--db=schej-it --gzip --archive=/var/backup/timeful.archive.gz
27
+
'';
28
+
24
29
clan.core.vars.generators.timeful = {
25
30
files.env = { };
26
31
runtimeInputs = [ pkgs.openssl ];
+3
configs/vaultwarden.nix
+3
configs/vaultwarden.nix
+8
configs/zitadel.nix
+8
configs/zitadel.nix
···
203
203
];
204
204
};
205
205
206
+
# users, passkeys, IdP links, OIDC apps; its secrets are encrypted with the
207
+
# masterkey from clan vars
208
+
lass.backup.state.zitadel.preBackup = ''
209
+
mkdir -p /var/backup/postgresql
210
+
${pkgs.sudo}/bin/sudo -u postgres ${config.services.postgresql.package}/bin/pg_dump \
211
+
zitadel | ${pkgs.gzip}/bin/gzip > /var/backup/postgresql/zitadel.sql.gz
212
+
'';
213
+
206
214
# Zitadel serves the console, OIDC and its gRPC APIs on one port over h2c.
207
215
services.nginx.virtualHosts.${domain} = {
208
216
enableACME = true;
+147
modules/backup.nix
+147
modules/backup.nix
···
1
+
{
2
+
config,
3
+
lib,
4
+
pkgs,
5
+
...
6
+
}:
7
+
let
8
+
cfg = config.lass.backup;
9
+
states = lib.attrValues cfg.state;
10
+
key = config.clan.core.vars.generators.borgbackup.files."borgbackup.ssh.id25519".path;
11
+
12
+
# Runs every state's hook in its own script, keeps going when one fails
13
+
# and fails at the end if any did.
14
+
runHooks =
15
+
hook:
16
+
pkgs.writeShellScript "backup-${hook}" ''
17
+
failed=()
18
+
${lib.concatMapStrings (state: ''
19
+
echo "backup: ${hook} ${state.name}"
20
+
${pkgs.writeShellScript "backup-${hook}-${state.name}" ''
21
+
set -xeo pipefail
22
+
${state.${hook}}
23
+
''} || failed+=(${lib.escapeShellArg state.name})
24
+
'') (lib.filter (state: state.${hook} != null) states)}
25
+
if [ ''${#failed[@]} -gt 0 ]; then
26
+
echo "backup: ${hook} failed for: ''${failed[*]}" >&2
27
+
exit 1
28
+
fi
29
+
'';
30
+
in
31
+
{
32
+
options.lass.backup = {
33
+
repo = lib.mkOption {
34
+
type = lib.types.nullOr lib.types.str;
35
+
default = null;
36
+
example = "user@backup.example.com:/./machine";
37
+
description = ''
38
+
Borg repository the machine backs up to. Without it no backup job is
39
+
generated, so modules can declare their state everywhere.
40
+
'';
41
+
};
42
+
43
+
sshPort = lib.mkOption {
44
+
type = lib.types.port;
45
+
default = 22;
46
+
};
47
+
48
+
knownHosts = lib.mkOption {
49
+
type = lib.types.lines;
50
+
description = "known_hosts lines pinning the host key of the repo's server.";
51
+
};
52
+
53
+
state = lib.mkOption {
54
+
default = { };
55
+
description = ''
56
+
Backup needs of a service, declared next to it. Dumps go to
57
+
/var/backup, which is always backed up.
58
+
'';
59
+
type = lib.types.attrsOf (
60
+
lib.types.submodule (
61
+
{ name, ... }:
62
+
{
63
+
options = {
64
+
name = lib.mkOption {
65
+
type = lib.types.str;
66
+
default = name;
67
+
};
68
+
folders = lib.mkOption {
69
+
type = lib.types.listOf lib.types.str;
70
+
default = [ ];
71
+
};
72
+
exclude = lib.mkOption {
73
+
type = lib.types.listOf lib.types.str;
74
+
default = [ ];
75
+
description = "borg exclude patterns.";
76
+
};
77
+
preBackup = lib.mkOption {
78
+
type = lib.types.nullOr lib.types.lines;
79
+
default = null;
80
+
description = ''
81
+
Runs before the backup inside the read-only borg unit (as
82
+
root), e.g. to dump a database to /var/backup. If it fails,
83
+
no backup is made and the job fails.
84
+
'';
85
+
};
86
+
postBackup = lib.mkOption {
87
+
type = lib.types.nullOr lib.types.lines;
88
+
default = null;
89
+
description = ''
90
+
Runs as root after the backup, also when it failed, e.g. to
91
+
start a service stopped by preBackup.
92
+
'';
93
+
};
94
+
readWritePaths = lib.mkOption {
95
+
type = lib.types.listOf lib.types.str;
96
+
default = [ ];
97
+
description = "Paths preBackup needs to write to.";
98
+
};
99
+
};
100
+
}
101
+
)
102
+
);
103
+
};
104
+
};
105
+
106
+
config = lib.mkIf (cfg.repo != null) {
107
+
clan.core.vars.generators.borgbackup = {
108
+
files."borgbackup.ssh.id25519" = { };
109
+
files."borgbackup.ssh.id25519.pub".secret = false;
110
+
runtimeInputs = with pkgs; [
111
+
coreutils
112
+
openssh
113
+
];
114
+
script = ''
115
+
ssh-keygen -t ed25519 -N "" -f $out/borgbackup.ssh.id25519
116
+
'';
117
+
};
118
+
119
+
systemd.tmpfiles.rules = [ "d /var/backup 0700 root root -" ];
120
+
121
+
# Archives are named and pruned by <hostname>-<job>; another job name
122
+
# would leave the existing archives unpruned.
123
+
services.borgbackup.jobs.hetzner = {
124
+
inherit (cfg) repo;
125
+
paths = lib.unique ([ "/var/backup" ] ++ lib.concatMap (state: state.folders) states);
126
+
exclude = lib.unique ([ "*.pyc" ] ++ lib.concatMap (state: state.exclude) states);
127
+
readWritePaths = lib.unique (
128
+
[ "/var/backup" ] ++ lib.concatMap (state: state.readWritePaths) states
129
+
);
130
+
preHook = "${runHooks "preBackup"}";
131
+
encryption.mode = "none";
132
+
compression = "auto,zstd";
133
+
startAt = "*-*-* 03:00:00";
134
+
environment.BORG_RSH = "ssh -oPort=${toString cfg.sshPort} -oUserKnownHostsFile=${pkgs.writeText "backup-known-hosts" cfg.knownHosts} -i ${key}";
135
+
prune.keep = {
136
+
within = "1d";
137
+
daily = 7;
138
+
weekly = 4;
139
+
monthly = 0;
140
+
};
141
+
};
142
+
143
+
systemd.services.borgbackup-job-hetzner.serviceConfig.ExecStopPost = lib.mkIf (lib.any (
144
+
state: state.postBackup != null
145
+
) states) [ "+${runHooks "postBackup"}" ];
146
+
};
147
+
}
+1
modules/default.nix
+1
modules/default.nix
+15
modules/parley.nix
+15
modules/parley.nix
···
137
137
PrivateDevices = true;
138
138
};
139
139
};
140
+
141
+
# identity.key also lives in clan vars; the SQLite databases are dumped
142
+
# consistently instead of copied.
143
+
lass.backup.state.parley = {
144
+
folders = [ "/var/lib/parley" ];
145
+
exclude = [ "/var/lib/parley/*.db*" ];
146
+
preBackup = ''
147
+
mkdir -p /var/backup/parley
148
+
for db in parley history; do
149
+
${pkgs.sqlite}/bin/sqlite3 -cmd ".timeout 60000" "/var/lib/parley/$db.db" ".backup /var/backup/parley/$db.db"
150
+
done
151
+
'';
152
+
# WAL-mode SQLite readers need to write the -shm file
153
+
readWritePaths = [ "/var/lib/parley" ];
154
+
};
140
155
};
141
156
}
+2
configs/paste/default.nix
+2
configs/paste/default.nix
+6
-113
machines/neoprism/backup.nix
+6
-113
machines/neoprism/backup.nix
···
1
-
{ config, pkgs, ... }:
2
1
{
3
-
clan.core.vars.generators.borgbackup = {
4
-
files."borgbackup.ssh.id25519" = { };
5
-
files."borgbackup.ssh.id25519.pub".secret = false;
6
-
runtimeInputs = with pkgs; [
7
-
coreutils
8
-
openssh
9
-
];
10
-
script = ''
11
-
ssh-keygen -t ed25519 -N "" -f $out/borgbackup.ssh.id25519
12
-
'';
13
-
};
14
-
15
-
# Ensure backup directory exists for pg_dump
16
-
systemd.tmpfiles.rules = [
17
-
"d /var/backup 0700 root root -"
18
-
"d /var/backup/postgresql 0700 root root -"
19
-
"d /var/backup/parley 0700 root root -"
20
-
];
21
-
22
-
services.borgbackup.jobs.hetzner = {
23
-
paths = [
24
-
"/var/backup"
25
-
"/var/lib/bitwarden_rs"
26
-
"/var/lib/htgen-cyberlocker"
27
-
"/var/lib/htgen-paste"
28
-
"/var/lib/matrix-synapse/media_store/local_content"
29
-
"/var/lib/radicale"
30
-
"/home/bot"
31
-
"/var/vmail"
32
-
"/var/dkim"
33
-
"/var/sieve"
34
-
"/var/lib/heisenbridge"
35
-
# identity.key also lives in clan vars; the SQLite databases are
36
-
# dumped consistently to /var/backup/parley by the preHook.
37
-
"/var/lib/parley"
38
-
# WBO boards (paint.lassul.us). DynamicUser + StateDirectory make
39
-
# /var/lib/whitebophir only a symlink, so back up the real directory.
40
-
"/var/lib/private/whitebophir"
41
-
# paperless (paperless.lassul.us) documents, index and classifier; the
42
-
# SQLite database is dumped consistently to /var/backup by the preHook.
43
-
"/var/lib/paperless"
44
-
];
45
-
exclude = [
46
-
"*.pyc"
47
-
"/home/bot/.cache"
48
-
"/home/bot/.nix-defexpr"
49
-
"/home/bot/.nix-profile"
50
-
"/var/lib/parley/*.db*"
51
-
"/var/lib/paperless/db.sqlite3*"
52
-
];
2
+
# Hetzner storage box; what gets backed up is declared next to each service
3
+
# (lass.backup.state).
4
+
lass.backup = {
53
5
repo = "u550643@u550643.your-storagebox.de:/./neoprism";
54
-
encryption.mode = "none";
55
-
compression = "auto,zstd";
56
-
startAt = "*-*-* 03:00:00";
57
-
environment.BORG_RSH =
58
-
let
59
-
knownHostsFile = pkgs.writeText "storagebox-known-hosts" ''
60
-
[u550643.your-storagebox.de]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs
61
-
'';
62
-
in
63
-
"ssh -oPort=23 -oUserKnownHostsFile=${knownHostsFile} -i ${
64
-
config.clan.core.vars.generators.borgbackup.files."borgbackup.ssh.id25519".path
65
-
}";
66
-
67
-
readWritePaths = [
68
-
"/var/backup"
69
-
"/var/lib/hedgedoc"
70
-
# WAL-mode SQLite readers need to write the -shm file
71
-
"/var/lib/parley"
72
-
"/var/lib/paperless"
73
-
];
74
-
75
-
preHook = ''
76
-
set -x
77
-
# Dump matrix-synapse PostgreSQL database (14GB, compresses well)
78
-
${pkgs.sudo}/bin/sudo -u postgres ${config.services.postgresql.package}/bin/pg_dump \
79
-
matrix-synapse | ${pkgs.gzip}/bin/gzip > /var/backup/postgresql/matrix-synapse.sql.gz
80
-
81
-
# Dump bridge PostgreSQL databases — portal mappings, sessions, tokens.
82
-
# Losing these = relogin everywhere + every existing Matrix room orphaned.
83
-
for db in mautrix-whatsapp mautrix-signal mautrix-telegram mautrix-discord; do
84
-
${pkgs.sudo}/bin/sudo -u postgres ${config.services.postgresql.package}/bin/pg_dump \
85
-
"$db" | ${pkgs.gzip}/bin/gzip > "/var/backup/postgresql/$db.sql.gz"
86
-
done
87
-
88
-
# Dump zitadel (id.lassul.us) PostgreSQL database: users, passkeys,
89
-
# IdP links, OIDC apps. Its secrets are encrypted with the masterkey
90
-
# from clan vars.
91
-
${pkgs.sudo}/bin/sudo -u postgres ${config.services.postgresql.package}/bin/pg_dump \
92
-
zitadel | ${pkgs.gzip}/bin/gzip > /var/backup/postgresql/zitadel.sql.gz
93
-
94
-
# Dump hedgedoc SQLite database safely. It uses a rollback journal and
95
-
# writes often during the day, so wait for its locks instead of failing.
96
-
${pkgs.sqlite}/bin/sqlite3 -cmd ".timeout 60000" /var/lib/hedgedoc/db.sqlite ".backup /var/backup/hedgedoc.sqlite"
97
-
98
-
# Dump parley accounts + history SQLite databases safely
99
-
for db in parley history; do
100
-
${pkgs.sqlite}/bin/sqlite3 -cmd ".timeout 60000" "/var/lib/parley/$db.db" ".backup /var/backup/parley/$db.db"
101
-
done
102
-
103
-
# Dump paperless SQLite database safely
104
-
${pkgs.sqlite}/bin/sqlite3 -cmd ".timeout 60000" /var/lib/paperless/db.sqlite3 ".backup /var/backup/paperless.sqlite3"
105
-
106
-
# Dump timeful polls (when.lassul.us) from MongoDB
107
-
${pkgs.mongodb-tools}/bin/mongodump --uri=mongodb://127.0.0.1:27017 \
108
-
--db=schej-it --gzip --archive=/var/backup/timeful.archive.gz
6
+
sshPort = 23;
7
+
knownHosts = ''
8
+
[u550643.your-storagebox.de]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs
109
9
'';
110
-
111
-
prune.keep = {
112
-
within = "1d";
113
-
daily = 7;
114
-
weekly = 4;
115
-
monthly = 0;
116
-
};
117
10
};
118
11
}
+10
machines/neoprism/config.nix
+10
machines/neoprism/config.nix
···
147
147
ip saddr 95.217.192.52 accept
148
148
'';
149
149
150
+
# not defined by any module
151
+
lass.backup.state.bot-home = {
152
+
folders = [ "/home/bot" ];
153
+
exclude = [
154
+
"/home/bot/.cache"
155
+
"/home/bot/.nix-defexpr"
156
+
"/home/bot/.nix-profile"
157
+
];
158
+
};
159
+
150
160
# tank/radicle (radicle seed storage) is created by disko at install; on an
151
161
# already-provisioned host it must be created before deploying:
152
162
# zfs create -o mountpoint=/var/lib/radicle -o quota=80G tank/radicle
+10
configs/services/matrix/default.nix
+10
configs/services/matrix/default.nix
···
120
120
}
121
121
];
122
122
};
123
+
124
+
lass.backup.state.matrix-synapse = {
125
+
folders = [ "/var/lib/matrix-synapse/media_store/local_content" ];
126
+
# 14GB, compresses well
127
+
preBackup = ''
128
+
mkdir -p /var/backup/postgresql
129
+
${pkgs.sudo}/bin/sudo -u postgres ${config.services.postgresql.package}/bin/pg_dump \
130
+
matrix-synapse | ${pkgs.gzip}/bin/gzip > /var/backup/postgresql/matrix-synapse.sql.gz
131
+
'';
132
+
};
123
133
}
+10
configs/services/pad/module.nix
+10
configs/services/pad/module.nix
···
80
80
systemd.services.hedgedoc.serviceConfig.Restart = "always";
81
81
systemd.services.hedgedoc.environment.NODE_OPTIONS = "--max-old-space-size=512";
82
82
83
+
lass.backup.state.hedgedoc = {
84
+
folders = [ config.services.hedgedoc.settings.uploadsPath ];
85
+
# The database uses a rollback journal and is written often during the
86
+
# day, so the dump waits for its locks instead of failing.
87
+
preBackup = ''
88
+
${pkgs.sqlite}/bin/sqlite3 -cmd ".timeout 60000" /var/lib/hedgedoc/db.sqlite ".backup /var/backup/hedgedoc.sqlite"
89
+
'';
90
+
readWritePaths = [ "/var/lib/hedgedoc" ];
91
+
};
92
+
83
93
systemd.services.hedgedoc.serviceConfig.EnvironmentFile = [
84
94
config.clan.core.vars.generators.hedgedoc.files."hedgedoc.env".path
85
95
];
+3
configs/services/paint/default.nix
+3
configs/services/paint/default.nix
+12
configs/services/paperless/default.nix
+12
configs/services/paperless/default.nix
···
39
39
};
40
40
};
41
41
42
+
# documents, index and classifier; the SQLite database is dumped
43
+
# consistently instead of copied
44
+
lass.backup.state.paperless = {
45
+
folders = [ "/var/lib/paperless" ];
46
+
exclude = [ "/var/lib/paperless/db.sqlite3*" ];
47
+
preBackup = ''
48
+
${pkgs.sqlite}/bin/sqlite3 -cmd ".timeout 60000" /var/lib/paperless/db.sqlite3 ".backup /var/backup/paperless.sqlite3"
49
+
'';
50
+
# WAL-mode SQLite readers need to write the -shm file
51
+
readWritePaths = [ "/var/lib/paperless" ];
52
+
};
53
+
42
54
# Zitadel (id.lassul.us) login, only for users with a grant on the project.
43
55
# zitadel-reconcile writes the client id/secret to zitadelEnv.
44
56
lass.zitadel.projects.paperless = {
+2
configs/services/matrix/bridges/heisenbridge.nix
+2
configs/services/matrix/bridges/heisenbridge.nix
+9
-1
configs/services/matrix/bridges/mautrix-discord.nix
+9
-1
configs/services/matrix/bridges/mautrix-discord.nix
···
1
-
{ ... }:
1
+
{ config, pkgs, ... }:
2
2
{
3
3
# The upstream module creates /var/lib/mautrix-discord with mode 0770 owned
4
4
# by mautrix-discord:mautrix-discord, while matrix-synapse only gets the
···
62
62
}
63
63
];
64
64
};
65
+
66
+
# portal mappings, sessions, tokens: losing them means relogin and every
67
+
# existing Matrix room orphaned
68
+
lass.backup.state.mautrix-discord.preBackup = ''
69
+
mkdir -p /var/backup/postgresql
70
+
${pkgs.sudo}/bin/sudo -u postgres ${config.services.postgresql.package}/bin/pg_dump \
71
+
mautrix-discord | ${pkgs.gzip}/bin/gzip > /var/backup/postgresql/mautrix-discord.sql.gz
72
+
'';
65
73
}
+9
-1
configs/services/matrix/bridges/mautrix-signal.nix
+9
-1
configs/services/matrix/bridges/mautrix-signal.nix
···
1
-
{ pkgs, ... }:
1
+
{ config, pkgs, ... }:
2
2
{
3
3
services.mautrix-signal = {
4
4
enable = true;
···
65
65
}
66
66
];
67
67
};
68
+
69
+
# portal mappings, sessions, tokens: losing them means relogin and every
70
+
# existing Matrix room orphaned
71
+
lass.backup.state.mautrix-signal.preBackup = ''
72
+
mkdir -p /var/backup/postgresql
73
+
${pkgs.sudo}/bin/sudo -u postgres ${config.services.postgresql.package}/bin/pg_dump \
74
+
mautrix-signal | ${pkgs.gzip}/bin/gzip > /var/backup/postgresql/mautrix-signal.sql.gz
75
+
'';
68
76
}
+14
-1
configs/services/matrix/bridges/mautrix-telegram.nix
+14
-1
configs/services/matrix/bridges/mautrix-telegram.nix
···
1
-
{ self, config, ... }:
1
+
{
2
+
self,
3
+
config,
4
+
pkgs,
5
+
...
6
+
}:
2
7
{
3
8
# mautrix-telegram is the Python (Telethon) bridge and pulls in python-olm
4
9
# → libolm 3.2.16, which nixpkgs marks insecure. The Go-based v2 bridges
···
62
67
}
63
68
];
64
69
};
70
+
71
+
# portal mappings, sessions, tokens: losing them means relogin and every
72
+
# existing Matrix room orphaned
73
+
lass.backup.state.mautrix-telegram.preBackup = ''
74
+
mkdir -p /var/backup/postgresql
75
+
${pkgs.sudo}/bin/sudo -u postgres ${config.services.postgresql.package}/bin/pg_dump \
76
+
mautrix-telegram | ${pkgs.gzip}/bin/gzip > /var/backup/postgresql/mautrix-telegram.sql.gz
77
+
'';
65
78
66
79
clan.core.vars.generators.mautrix-telegram = {
67
80
files.env = { };
+9
-1
configs/services/matrix/bridges/mautrix-whatsapp.nix
+9
-1
configs/services/matrix/bridges/mautrix-whatsapp.nix
···
1
-
{ pkgs, ... }:
1
+
{ config, pkgs, ... }:
2
2
{
3
3
services.mautrix-whatsapp = {
4
4
enable = true;
···
72
72
}
73
73
];
74
74
};
75
+
76
+
# portal mappings, sessions, tokens: losing them means relogin and every
77
+
# existing Matrix room orphaned
78
+
lass.backup.state.mautrix-whatsapp.preBackup = ''
79
+
mkdir -p /var/backup/postgresql
80
+
${pkgs.sudo}/bin/sudo -u postgres ${config.services.postgresql.package}/bin/pg_dump \
81
+
mautrix-whatsapp | ${pkgs.gzip}/bin/gzip > /var/backup/postgresql/mautrix-whatsapp.sql.gz
82
+
'';
75
83
}