This repository has no description
flix/ipfs: remove MFS browse-index (kubo daemon deadlock)
kubo 0.40.1 has a daemon-side MFS goroutine deadlock: the first
`ipfs files` write after each daemon start permanently wedges the global
MFS lock, so even a trivial `files mkdir` then hangs forever. This
silently parked the pin-watcher on its startup `files mkdir` and stopped
auto-pinning for ~4 days. It is not fixable client-side (timeout/flock/
serialize cannot release a lock held by a wedged daemon goroutine).
Rip out the browse-index entirely rather than gate it behind a toggle:
- drop the publish oneshot + timer (IPNS publish of the MFS root)
- drop all `ipfs files` ops from the watcher and reconcile, plus the
mfs()/mfs_add/mfs_rm/mark_dirty/relpath helpers, IPNS keygen, startup
mkdir, backfill, and the dirty-flag/mfs-lock machinery
- rename flix-index-reconcile -> ipfs-pin-reconcile (it is pin-only now)
Pinning is unaffected: the inotify watcher still `ipfs add --nocopy
--pin`s under the pin-add flock, and the hourly forward-sweep reconcile
still heals dropped inotify events.
author
lassulus
date
(Jun 26, 2026, 1:17 PM CEST)
commit
6a69b329
6a69b329c1ba76ecfdccfe5439843ab36936d885
parent
2c1c43da
2c1c43daae2130ff264add1ef00f95a6cde147d4
+29
-176
+29
-176
2configs/services/flix/ipfs.nix
+29
-176
2configs/services/flix/ipfs.nix
···
12
12
];
13
13
14
14
cidMapFile = "/var/lib/ipfs/cid-map.txt";
15
-
downloadRoot = "/var/lib/ipfs/download";
16
-
flixIndexMfs = "/flix-index";
17
-
dirtyFlag = "/var/lib/ipfs/flix-index.dirty";
18
-
ipnsNameFile = "/var/lib/ipfs/flix-index.name";
19
-
# Serializes `ipfs add --nocopy` between the pin-watcher and the
20
-
# reconcile sweep: two concurrent --nocopy adds of the same file corrupt
21
-
# the filestore into an incomplete DAG.
15
+
# Serializes `ipfs add --nocopy` between the pin-watcher and the reconcile
16
+
# sweep: two concurrent --nocopy adds of the same file corrupt the filestore
17
+
# into an incomplete DAG.
22
18
pinAddLock = "/var/lib/ipfs/pin-add.lock";
23
19
24
20
pinWatcherScript = pkgs.writers.writeBash "ipfs-pin-watcher" ''
···
27
23
IPFS="${pkgs.kubo}/bin/ipfs"
28
24
INOTIFYWAIT="${pkgs.inotify-tools}/bin/inotifywait"
29
25
CID_MAP="${cidMapFile}"
30
-
DOWNLOAD_ROOT="${downloadRoot}"
31
-
FLIX_INDEX="${flixIndexMfs}"
32
-
DIRTY_FLAG="${dirtyFlag}"
33
-
IPNS_NAME_FILE="${ipnsNameFile}"
34
26
PIN_LOCK="${pinAddLock}"
35
27
36
28
touch "$CID_MAP"
···
39
31
log() {
40
32
echo "[$(date -Iseconds)] $*"
41
33
}
42
-
43
-
mark_dirty() {
44
-
touch "$DIRTY_FLAG"
45
-
}
46
-
47
-
# strip $DOWNLOAD_ROOT/ prefix to get the relative path used inside MFS
48
-
relpath() {
49
-
echo "''${1#$DOWNLOAD_ROOT/}"
50
-
}
51
-
52
-
mfs_add() {
53
-
local cid="$1" path="$2"
54
-
local rel dir
55
-
rel=$(relpath "$path")
56
-
dir=$(${pkgs.coreutils}/bin/dirname "$rel")
57
-
# kubo's `files rm/cp/mkdir` write detailed errors to stdout, not
58
-
# stderr, so both FDs need redirection to keep backfill quiet.
59
-
$IPFS files mkdir -p "$FLIX_INDEX/$dir" >/dev/null 2>&1 || true
60
-
$IPFS files rm "$FLIX_INDEX/$rel" >/dev/null 2>&1 || true
61
-
$IPFS files cp "/ipfs/$cid" "$FLIX_INDEX/$rel" >/dev/null 2>&1 || true
62
-
mark_dirty
63
-
}
64
-
65
-
mfs_rm() {
66
-
local path="$1"
67
-
local rel
68
-
rel=$(relpath "$path")
69
-
$IPFS files rm "$FLIX_INDEX/$rel" >/dev/null 2>&1 || true
70
-
mark_dirty
71
-
}
72
-
73
-
# ensure IPNS key + MFS root exist (idempotent)
74
-
if ! $IPFS key list | ${pkgs.gnugrep}/bin/grep -qx flix-index; then
75
-
log "Generating flix-index IPNS key"
76
-
$IPFS key gen --type=ed25519 flix-index > "$IPNS_NAME_FILE" || true
77
-
fi
78
-
if [ -f "$IPNS_NAME_FILE" ]; then
79
-
log "flix-index IPNS name: $(cat "$IPNS_NAME_FILE")"
80
-
fi
81
-
$IPFS files mkdir -p "$FLIX_INDEX" >/dev/null 2>&1 || true
82
34
83
35
add_file() {
84
36
local path="$1"
···
96
48
log "Pinned $path -> $cid"
97
49
${pkgs.gnused}/bin/sed -i "\|^[^ ]* $path$|d" "$CID_MAP"
98
50
echo "$cid $path" >> "$CID_MAP"
99
-
mfs_add "$cid" "$path"
100
51
}
101
52
102
53
remove_file() {
···
104
55
cid=$(${pkgs.gawk}/bin/awk -v p="$path" '$0 ~ p {print $1; exit}' "$CID_MAP")
105
56
if [ -n "$cid" ]; then
106
57
log "Unpinning $cid ($path)"
107
-
# remove from MFS first so the implicit MFS pin doesn't keep it alive
108
-
mfs_rm "$path"
109
58
$IPFS pin rm "$cid" 2>/dev/null || true
110
59
${pkgs.gnused}/bin/sed -i "\|^[^ ]* $path$|d" "$CID_MAP"
111
60
else
···
113
62
fi
114
63
}
115
64
116
-
# initial sync
65
+
# initial sync: pin any file in the watch dirs not already in cid-map
117
66
log "Starting initial sync..."
118
67
for dir in ${lib.escapeShellArgs watchDirs}; do
119
68
if [ -d "$dir" ]; then
···
126
75
done
127
76
log "Initial sync complete"
128
77
129
-
# clean stale entries
78
+
# clean stale entries: unpin files that no longer exist on disk
130
79
log "Cleaning stale entries..."
131
80
while IFS=' ' read -r cid path; do
132
81
if [ ! -f "$path" ]; then
133
82
log "Stale entry: $path (CID: $cid)"
134
-
mfs_rm "$path"
135
83
$IPFS pin rm "$cid" 2>/dev/null || true
136
84
${pkgs.gnused}/bin/sed -i "\|^[^ ]* $path$|d" "$CID_MAP"
137
85
fi
138
86
done < "$CID_MAP"
139
87
log "Cleanup complete"
140
88
141
-
# backfill MFS index from cid-map.txt once (sentinel-guarded so we run
142
-
# exactly once per machine, independently of whether the initial sync
143
-
# happened to create any MFS entries incrementally).
144
-
BACKFILL_SENTINEL="/var/lib/ipfs/flix-index.backfilled"
145
-
if [ ! -f "$BACKFILL_SENTINEL" ]; then
146
-
log "Backfilling MFS index from cid-map.txt..."
147
-
n=0
148
-
while IFS=' ' read -r cid path; do
149
-
[ -f "$path" ] || continue
150
-
# skip entries already present (idempotent re-runs)
151
-
rel=$(relpath "$path")
152
-
if $IPFS files stat "$FLIX_INDEX/$rel" >/dev/null 2>&1; then
153
-
continue
154
-
fi
155
-
mfs_add "$cid" "$path"
156
-
n=$((n + 1))
157
-
done < "$CID_MAP"
158
-
touch "$BACKFILL_SENTINEL"
159
-
log "Backfill complete ($n entries added)"
160
-
fi
161
-
162
-
# add all files in a newly appeared directory
89
+
# pin all files in a newly appeared directory (covers the inotify
90
+
# recursive-watch race when a brand-new subdir is created + filled fast)
163
91
scan_new_dir() {
164
92
local dir="$1"
165
93
log "Scanning new directory: $dir"
···
193
121
done
194
122
'';
195
123
196
-
publishScript = pkgs.writers.writeBash "flix-index-publish" ''
197
-
set -efu
198
-
199
-
IPFS="${pkgs.kubo}/bin/ipfs"
200
-
CID_MAP="${cidMapFile}"
201
-
DOWNLOAD_ROOT="${downloadRoot}"
202
-
FLIX_INDEX="${flixIndexMfs}"
203
-
DIRTY_FLAG="${dirtyFlag}"
204
-
205
-
[ -f "$DIRTY_FLAG" ] || exit 0
206
-
207
-
# regenerate index.tsv from cid-map.txt
208
-
tmp=$(${pkgs.coreutils}/bin/mktemp)
209
-
printf 'cid\tpath\tsize\n' > "$tmp"
210
-
while IFS=' ' read -r cid path; do
211
-
[ -f "$path" ] || continue
212
-
rel="''${path#$DOWNLOAD_ROOT/}"
213
-
size=$(${pkgs.coreutils}/bin/stat -c %s "$path" 2>/dev/null || echo 0)
214
-
printf '%s\t%s\t%s\n' "$cid" "$rel" "$size" >> "$tmp"
215
-
done < "$CID_MAP"
216
-
217
-
$IPFS files rm "$FLIX_INDEX/index.tsv" >/dev/null 2>&1 || true
218
-
$IPFS files write --create --truncate "$FLIX_INDEX/index.tsv" < "$tmp"
219
-
${pkgs.coreutils}/bin/rm -f "$tmp"
220
-
221
-
root=$($IPFS files stat --hash "$FLIX_INDEX")
222
-
echo "Publishing $root to IPNS (flix-index)"
223
-
$IPFS name publish --key=flix-index --lifetime=48h --ttl=1m "/ipfs/$root"
224
-
# Announce the MFS root to the DHT so public gateways can find us.
225
-
# MFS directory blocks are only kept alive by an implicit pin that
226
-
# Provide.Strategy="pinned" ignores, so we provide them explicitly here.
227
-
echo "Announcing $root to DHT"
228
-
$IPFS routing provide "$root" >/dev/null 2>&1 || true
229
-
${pkgs.coreutils}/bin/rm -f "$DIRTY_FLAG"
230
-
'';
231
-
232
124
uploadServer = pkgs.writers.writePython3Bin "ipfs-upload-server" {
233
125
flakeIgnore = [ "E501" ];
234
126
} ./ipfs_upload.py;
235
127
236
-
# Periodic reconciliation, BOTH directions, so cid-map/MFS/pins stay in
237
-
# sync with the filesystem even when inotify drops events:
238
-
# reverse - drop cid-map entries / pins for files no longer on disk
239
-
# (e.g. queue overflow during bulk renames/deletes).
240
-
# forward - pin files present on disk but missing from cid-map. The
241
-
# pin-watcher loop is single-threaded and blocks 1-2 min on each
242
-
# multi-GB `ipfs add`; while blocked, a concurrent upload's
243
-
# CREATE,ISDIR + MOVED_TO for a brand-new game subdir can overflow
244
-
# the inotify queue and be dropped, leaving a fully-written file
245
-
# that is never pinned. The forward sweep heals that within an hour
246
-
# (without it a dropped event is permanent until a watcher restart
247
-
# re-runs the startup sync).
248
-
reconcileScript = pkgs.writers.writeBash "flix-index-reconcile" ''
128
+
# Periodic reconciliation so cid-map/pins stay in sync with the filesystem
129
+
# even when the inotify watcher drops events:
130
+
# reverse - unpin + drop cid-map entries for files no longer on disk.
131
+
# forward - pin files present on disk but missing from cid-map. The watcher
132
+
# loop is single-threaded and blocks 1-2 min on each multi-GB
133
+
# `ipfs add`; while blocked, a concurrent upload's CREATE,ISDIR +
134
+
# MOVED_TO for a brand-new subdir can overflow the inotify queue
135
+
# and be dropped, leaving a fully-written file never pinned. The
136
+
# forward sweep heals that within an hour.
137
+
reconcileScript = pkgs.writers.writeBash "ipfs-pin-reconcile" ''
249
138
set -efu
250
139
251
140
IPFS="${pkgs.kubo}/bin/ipfs"
252
141
CID_MAP="${cidMapFile}"
253
-
DOWNLOAD_ROOT="${downloadRoot}"
254
-
FLIX_INDEX="${flixIndexMfs}"
255
-
DIRTY_FLAG="${dirtyFlag}"
256
142
PIN_LOCK="${pinAddLock}"
257
143
258
144
[ -f "$CID_MAP" ] || exit 0
259
145
260
-
# Rebuild cid-map in one pass, collecting stale entries to clean up.
261
-
# Avoids sed regex issues with special chars like [ ] in paths.
262
-
# Small race with pin-watcher's concurrent appends — any line the
263
-
# watcher writes during this loop may be lost, but will be re-added
264
-
# on the next inotify event or watcher restart.
146
+
# Reverse: rebuild cid-map, unpinning entries whose file is gone. One pass
147
+
# avoids sed regex issues with special chars like [ ] in paths. Small race
148
+
# with the watcher's concurrent appends -- a line written during this loop
149
+
# may be lost, but is re-added on the next inotify event or watcher restart.
265
150
tmp=$(${pkgs.coreutils}/bin/mktemp)
266
151
removed=0
267
152
while IFS=' ' read -r cid path; do
···
269
154
if [ -e "$path" ]; then
270
155
echo "$cid $path" >> "$tmp"
271
156
else
272
-
rel="''${path#$DOWNLOAD_ROOT/}"
273
-
$IPFS files rm "$FLIX_INDEX/$rel" >/dev/null 2>&1 || true
274
157
$IPFS pin rm "$cid" >/dev/null 2>&1 || true
275
158
removed=$((removed + 1))
276
159
fi
···
280
163
${pkgs.coreutils}/bin/chmod 0644 "$tmp"
281
164
${pkgs.coreutils}/bin/mv "$tmp" "$CID_MAP"
282
165
echo "reconciled: removed $removed stale entries"
283
-
touch "$DIRTY_FLAG"
284
166
else
285
167
${pkgs.coreutils}/bin/rm -f "$tmp"
286
168
fi
287
169
288
-
# Forward sweep: pin on-disk files that inotify missed (see header).
289
-
# Only files older than 10 min, so the flock + this margin guarantee we
290
-
# never collide with the pin-watcher mid-add on a fresh upload (a file
291
-
# that old and still absent from cid-map was dropped, not in-flight).
170
+
# Forward: pin on-disk files missing from cid-map. Only files older than
171
+
# 10 min, so the flock + margin never collide with the watcher mid-add on a
172
+
# fresh upload (a file that old and still absent was dropped, not in-flight).
292
173
for dir in ${lib.escapeShellArgs watchDirs}; do
293
174
[ -d "$dir" ] || continue
294
175
${pkgs.findutils}/bin/find "$dir" -type f -mmin +10 | while read -r f; do
···
304
185
continue
305
186
}
306
187
echo "$cid $f" >> "$CID_MAP"
307
-
rel="''${f#$DOWNLOAD_ROOT/}"
308
-
$IPFS files mkdir -p "$FLIX_INDEX/$(${pkgs.coreutils}/bin/dirname "$rel")" \
309
-
>/dev/null 2>&1 || true
310
-
$IPFS files cp "/ipfs/$cid" "$FLIX_INDEX/$rel" >/dev/null 2>&1 || true
311
-
touch "$DIRTY_FLAG"
312
188
echo "reconcile: pinned missed file $f -> $cid"
313
189
done
314
190
done
···
418
294
};
419
295
};
420
296
421
-
systemd.services.flix-index-publish = {
422
-
description = "Publish flix MFS index to IPNS";
423
-
after = [ "ipfs.service" ];
424
-
requires = [ "ipfs.service" ];
425
-
serviceConfig = {
426
-
Type = "oneshot";
427
-
Environment = [ "IPFS_PATH=/var/lib/ipfs" ];
428
-
ExecStart = publishScript;
429
-
User = config.services.kubo.user;
430
-
Group = config.services.kubo.group;
431
-
};
432
-
};
433
-
434
-
systemd.timers.flix-index-publish = {
435
-
description = "Publish flix MFS index to IPNS periodically";
436
-
wantedBy = [ "timers.target" ];
437
-
timerConfig = {
438
-
OnBootSec = "5min";
439
-
OnUnitActiveSec = "2min";
440
-
Unit = "flix-index-publish.service";
441
-
};
442
-
};
443
-
444
-
systemd.services.flix-index-reconcile = {
445
-
description = "Reconcile flix MFS index / cid-map against filesystem";
297
+
systemd.services.ipfs-pin-reconcile = {
298
+
description = "Reconcile pins / cid-map against the filesystem";
446
299
after = [ "ipfs.service" ];
447
300
requires = [ "ipfs.service" ];
448
301
serviceConfig = {
···
454
307
};
455
308
};
456
309
457
-
systemd.timers.flix-index-reconcile = {
458
-
description = "Reconcile flix MFS index hourly";
310
+
systemd.timers.ipfs-pin-reconcile = {
311
+
description = "Reconcile pins / cid-map hourly";
459
312
wantedBy = [ "timers.target" ];
460
313
timerConfig = {
461
314
OnBootSec = "15min";
462
315
OnUnitActiveSec = "1h";
463
-
Unit = "flix-index-reconcile.service";
316
+
Unit = "ipfs-pin-reconcile.service";
464
317
};
465
318
};
466
319