This repository has no description
2configs.services.flix.ipfs: publish pin index to IPNS as MFS tree + index.tsv
author
lassulus
date
(Apr 9, 2026, 11:11 AM +0100)
commit
8403de27
8403de277cab1a35e17494b597b412a5254b0b05
parent
6eb9b469
6eb9b46983f10b23bed4252a9e1164f5d6451a1c
+132
+132
2configs/services/flix/ipfs.nix
+132
2configs/services/flix/ipfs.nix
···
12
12
];
13
13
14
14
cidMapFile = "/var/lib/ipfs/cid-map.txt";
15
+
downloadRoot = "/var/lib/ipfs/download";
16
+
flixIndexMfs = "/flix-index";
17
+
dirtyFlag = "/var/lib/ipfs/flix-index.dirty";
18
+
ipnsNameFile = "/var/lib/ipfs/flix-index.name";
15
19
16
20
pinWatcherScript = pkgs.writers.writeBash "ipfs-pin-watcher" ''
17
21
set -efu
···
19
23
IPFS="${pkgs.kubo}/bin/ipfs"
20
24
INOTIFYWAIT="${pkgs.inotify-tools}/bin/inotifywait"
21
25
CID_MAP="${cidMapFile}"
26
+
DOWNLOAD_ROOT="${downloadRoot}"
27
+
FLIX_INDEX="${flixIndexMfs}"
28
+
DIRTY_FLAG="${dirtyFlag}"
29
+
IPNS_NAME_FILE="${ipnsNameFile}"
22
30
23
31
touch "$CID_MAP"
24
32
25
33
log() {
26
34
echo "[$(date -Iseconds)] $*"
27
35
}
36
+
37
+
mark_dirty() {
38
+
touch "$DIRTY_FLAG"
39
+
}
40
+
41
+
# strip $DOWNLOAD_ROOT/ prefix to get the relative path used inside MFS
42
+
relpath() {
43
+
echo "''${1#$DOWNLOAD_ROOT/}"
44
+
}
45
+
46
+
mfs_add() {
47
+
local cid="$1" path="$2"
48
+
local rel dir
49
+
rel=$(relpath "$path")
50
+
dir=$(${pkgs.coreutils}/bin/dirname "$rel")
51
+
# kubo's `files rm/cp/mkdir` write detailed errors to stdout, not
52
+
# stderr, so both FDs need redirection to keep backfill quiet.
53
+
$IPFS files mkdir -p "$FLIX_INDEX/$dir" >/dev/null 2>&1 || true
54
+
$IPFS files rm "$FLIX_INDEX/$rel" >/dev/null 2>&1 || true
55
+
$IPFS files cp "/ipfs/$cid" "$FLIX_INDEX/$rel" >/dev/null 2>&1 || true
56
+
mark_dirty
57
+
}
58
+
59
+
mfs_rm() {
60
+
local path="$1"
61
+
local rel
62
+
rel=$(relpath "$path")
63
+
$IPFS files rm "$FLIX_INDEX/$rel" >/dev/null 2>&1 || true
64
+
mark_dirty
65
+
}
66
+
67
+
# ensure IPNS key + MFS root exist (idempotent)
68
+
if ! $IPFS key list | ${pkgs.gnugrep}/bin/grep -qx flix-index; then
69
+
log "Generating flix-index IPNS key"
70
+
$IPFS key gen --type=ed25519 flix-index > "$IPNS_NAME_FILE" || true
71
+
fi
72
+
if [ -f "$IPNS_NAME_FILE" ]; then
73
+
log "flix-index IPNS name: $(cat "$IPNS_NAME_FILE")"
74
+
fi
75
+
$IPFS files mkdir -p "$FLIX_INDEX" >/dev/null 2>&1 || true
28
76
29
77
add_file() {
30
78
local path="$1"
···
41
89
log "Pinned $path -> $cid"
42
90
${pkgs.gnused}/bin/sed -i "\|^[^ ]* $path$|d" "$CID_MAP"
43
91
echo "$cid $path" >> "$CID_MAP"
92
+
mfs_add "$cid" "$path"
44
93
}
45
94
46
95
remove_file() {
···
48
97
cid=$(${pkgs.gawk}/bin/awk -v p="$path" '$0 ~ p {print $1; exit}' "$CID_MAP")
49
98
if [ -n "$cid" ]; then
50
99
log "Unpinning $cid ($path)"
100
+
# remove from MFS first so the implicit MFS pin doesn't keep it alive
101
+
mfs_rm "$path"
51
102
$IPFS pin rm "$cid" 2>/dev/null || true
52
103
${pkgs.gnused}/bin/sed -i "\|^[^ ]* $path$|d" "$CID_MAP"
53
104
else
···
73
124
while IFS=' ' read -r cid path; do
74
125
if [ ! -f "$path" ]; then
75
126
log "Stale entry: $path (CID: $cid)"
127
+
mfs_rm "$path"
76
128
$IPFS pin rm "$cid" 2>/dev/null || true
77
129
${pkgs.gnused}/bin/sed -i "\|^[^ ]* $path$|d" "$CID_MAP"
78
130
fi
79
131
done < "$CID_MAP"
80
132
log "Cleanup complete"
133
+
134
+
# backfill MFS index from cid-map.txt once (sentinel-guarded so we run
135
+
# exactly once per machine, independently of whether the initial sync
136
+
# happened to create any MFS entries incrementally).
137
+
BACKFILL_SENTINEL="/var/lib/ipfs/flix-index.backfilled"
138
+
if [ ! -f "$BACKFILL_SENTINEL" ]; then
139
+
log "Backfilling MFS index from cid-map.txt..."
140
+
n=0
141
+
while IFS=' ' read -r cid path; do
142
+
[ -f "$path" ] || continue
143
+
# skip entries already present (idempotent re-runs)
144
+
rel=$(relpath "$path")
145
+
if $IPFS files stat "$FLIX_INDEX/$rel" >/dev/null 2>&1; then
146
+
continue
147
+
fi
148
+
mfs_add "$cid" "$path"
149
+
n=$((n + 1))
150
+
done < "$CID_MAP"
151
+
touch "$BACKFILL_SENTINEL"
152
+
log "Backfill complete ($n entries added)"
153
+
fi
81
154
82
155
# add all files in a newly appeared directory
83
156
scan_new_dir() {
···
111
184
;;
112
185
esac
113
186
done
187
+
'';
188
+
189
+
publishScript = pkgs.writers.writeBash "flix-index-publish" ''
190
+
set -efu
191
+
192
+
IPFS="${pkgs.kubo}/bin/ipfs"
193
+
CID_MAP="${cidMapFile}"
194
+
DOWNLOAD_ROOT="${downloadRoot}"
195
+
FLIX_INDEX="${flixIndexMfs}"
196
+
DIRTY_FLAG="${dirtyFlag}"
197
+
198
+
[ -f "$DIRTY_FLAG" ] || exit 0
199
+
200
+
# regenerate index.tsv from cid-map.txt
201
+
tmp=$(${pkgs.coreutils}/bin/mktemp)
202
+
printf 'cid\tpath\tsize\n' > "$tmp"
203
+
while IFS=' ' read -r cid path; do
204
+
[ -f "$path" ] || continue
205
+
rel="''${path#$DOWNLOAD_ROOT/}"
206
+
size=$(${pkgs.coreutils}/bin/stat -c %s "$path" 2>/dev/null || echo 0)
207
+
printf '%s\t%s\t%s\n' "$cid" "$rel" "$size" >> "$tmp"
208
+
done < "$CID_MAP"
209
+
210
+
$IPFS files rm "$FLIX_INDEX/index.tsv" >/dev/null 2>&1 || true
211
+
$IPFS files write --create --truncate "$FLIX_INDEX/index.tsv" < "$tmp"
212
+
${pkgs.coreutils}/bin/rm -f "$tmp"
213
+
214
+
root=$($IPFS files stat --hash "$FLIX_INDEX")
215
+
echo "Publishing $root to IPNS (flix-index)"
216
+
$IPFS name publish --key=flix-index --lifetime=48h --ttl=1m "/ipfs/$root"
217
+
# Announce the MFS root to the DHT so public gateways can find us.
218
+
# MFS directory blocks are only kept alive by an implicit pin that
219
+
# Provide.Strategy="pinned" ignores, so we provide them explicitly here.
220
+
echo "Announcing $root to DHT"
221
+
$IPFS routing provide "$root" >/dev/null 2>&1 || true
222
+
${pkgs.coreutils}/bin/rm -f "$DIRTY_FLAG"
114
223
'';
115
224
in
116
225
{
···
180
289
User = config.services.kubo.user;
181
290
Group = config.services.kubo.group;
182
291
SupplementaryGroups = [ "radio_container" ];
292
+
};
293
+
};
294
+
295
+
systemd.services.flix-index-publish = {
296
+
description = "Publish flix MFS index to IPNS";
297
+
after = [ "ipfs.service" ];
298
+
requires = [ "ipfs.service" ];
299
+
serviceConfig = {
300
+
Type = "oneshot";
301
+
Environment = [ "IPFS_PATH=/var/lib/ipfs" ];
302
+
ExecStart = publishScript;
303
+
User = config.services.kubo.user;
304
+
Group = config.services.kubo.group;
305
+
};
306
+
};
307
+
308
+
systemd.timers.flix-index-publish = {
309
+
description = "Publish flix MFS index to IPNS periodically";
310
+
wantedBy = [ "timers.target" ];
311
+
timerConfig = {
312
+
OnBootSec = "5min";
313
+
OnUnitActiveSec = "2min";
314
+
Unit = "flix-index-publish.service";
183
315
};
184
316
};
185
317