This repository has no description
ipfs-pin-watcher: forward reconcile sweep + flock to heal dropped events
The pin-watcher loop is single-threaded and blocks 1-2 min on each multi-GB
`ipfs add`. While blocked, a concurrent upload's CREATE,ISDIR + MOVED_TO for a
brand-new game subdir can overflow the inotify queue and be dropped, leaving a
fully-written file that is never pinned (gateway 504s on it forever). The
hourly reconcile only did reverse cleanup (drop entries for vanished files), so
a dropped event was permanent until a watcher restart re-ran the startup sync.
Add a forward sweep to the reconcile: pin on-disk files older than 10 min that
are missing from cid-map. Guard both the watcher's add_file and the sweep with
a shared flock (pin-add.lock) so two concurrent --nocopy adds of the same file
can never corrupt the filestore DAG. flock auto-releases on process death/OOM/
reboot, and an add interrupted mid-pin never writes its cid-map line, so the
sweep re-adds and completes it on the next pass.
author
lassulus
date
(Jun 26, 2026, 1:05 AM CEST)
commit
2c1c43da
2c1c43daae2130ff264add1ef00f95a6cde147d4
parent
b7ed12be
b7ed12be1bdf5c021861880ea059bb4f2cbbe8e8
+48
-5
+48
-5
2configs/services/flix/ipfs.nix
+48
-5
2configs/services/flix/ipfs.nix
···
16
16
flixIndexMfs = "/flix-index";
17
17
dirtyFlag = "/var/lib/ipfs/flix-index.dirty";
18
18
ipnsNameFile = "/var/lib/ipfs/flix-index.name";
19
+
# Serializes `ipfs add --nocopy` between the pin-watcher and the
20
+
# reconcile sweep: two concurrent --nocopy adds of the same file corrupt
21
+
# the filestore into an incomplete DAG.
22
+
pinAddLock = "/var/lib/ipfs/pin-add.lock";
19
23
20
24
pinWatcherScript = pkgs.writers.writeBash "ipfs-pin-watcher" ''
21
25
set -efu
···
27
31
FLIX_INDEX="${flixIndexMfs}"
28
32
DIRTY_FLAG="${dirtyFlag}"
29
33
IPNS_NAME_FILE="${ipnsNameFile}"
34
+
PIN_LOCK="${pinAddLock}"
30
35
31
36
touch "$CID_MAP"
32
37
${pkgs.coreutils}/bin/chmod 0644 "$CID_MAP"
···
83
88
esac
84
89
85
90
log "Adding $path"
86
-
cid=$($IPFS add --nocopy --pin --quieter "$path" 2>/dev/null) || {
91
+
cid=$(${pkgs.util-linux}/bin/flock "$PIN_LOCK" \
92
+
$IPFS add --nocopy --pin --quieter "$path" 2>/dev/null) || {
87
93
log "Failed to add $path"
88
94
return 0
89
95
}
···
223
229
${pkgs.coreutils}/bin/rm -f "$DIRTY_FLAG"
224
230
'';
225
231
226
-
# Periodic reconciliation: walks cid-map.txt and repairs stale state for
227
-
# files that no longer exist on disk. Protects against inotify event
228
-
# drops (e.g. queue overflow during bulk renames/deletes) and any other
229
-
# path that leaves cid-map/MFS out of sync with the filesystem.
230
232
uploadServer = pkgs.writers.writePython3Bin "ipfs-upload-server" {
231
233
flakeIgnore = [ "E501" ];
232
234
} ./ipfs_upload.py;
233
235
236
+
# Periodic reconciliation, BOTH directions, so cid-map/MFS/pins stay in
237
+
# sync with the filesystem even when inotify drops events:
238
+
# reverse - drop cid-map entries / pins for files no longer on disk
239
+
# (e.g. queue overflow during bulk renames/deletes).
240
+
# forward - pin files present on disk but missing from cid-map. The
241
+
# pin-watcher loop is single-threaded and blocks 1-2 min on each
242
+
# multi-GB `ipfs add`; while blocked, a concurrent upload's
243
+
# CREATE,ISDIR + MOVED_TO for a brand-new game subdir can overflow
244
+
# the inotify queue and be dropped, leaving a fully-written file
245
+
# that is never pinned. The forward sweep heals that within an hour
246
+
# (without it a dropped event is permanent until a watcher restart
247
+
# re-runs the startup sync).
234
248
reconcileScript = pkgs.writers.writeBash "flix-index-reconcile" ''
235
249
set -efu
236
250
···
239
253
DOWNLOAD_ROOT="${downloadRoot}"
240
254
FLIX_INDEX="${flixIndexMfs}"
241
255
DIRTY_FLAG="${dirtyFlag}"
256
+
PIN_LOCK="${pinAddLock}"
242
257
243
258
[ -f "$CID_MAP" ] || exit 0
244
259
···
269
284
else
270
285
${pkgs.coreutils}/bin/rm -f "$tmp"
271
286
fi
287
+
288
+
# Forward sweep: pin on-disk files that inotify missed (see header).
289
+
# Only files older than 10 min, so the flock + this margin guarantee we
290
+
# never collide with the pin-watcher mid-add on a fresh upload (a file
291
+
# that old and still absent from cid-map was dropped, not in-flight).
292
+
for dir in ${lib.escapeShellArgs watchDirs}; do
293
+
[ -d "$dir" ] || continue
294
+
${pkgs.findutils}/bin/find "$dir" -type f -mmin +10 | while read -r f; do
295
+
case "$(${pkgs.coreutils}/bin/basename "$f")" in
296
+
.* | *.part | *.tmp | *.synced.* | *.aria2) continue ;;
297
+
esac
298
+
if ${pkgs.gnugrep}/bin/grep -q " $f$" "$CID_MAP" 2>/dev/null; then
299
+
continue
300
+
fi
301
+
cid=$(${pkgs.util-linux}/bin/flock "$PIN_LOCK" \
302
+
$IPFS add --nocopy --pin --quieter "$f" 2>/dev/null) || {
303
+
echo "reconcile: failed to add $f"
304
+
continue
305
+
}
306
+
echo "$cid $f" >> "$CID_MAP"
307
+
rel="''${f#$DOWNLOAD_ROOT/}"
308
+
$IPFS files mkdir -p "$FLIX_INDEX/$(${pkgs.coreutils}/bin/dirname "$rel")" \
309
+
>/dev/null 2>&1 || true
310
+
$IPFS files cp "/ipfs/$cid" "$FLIX_INDEX/$rel" >/dev/null 2>&1 || true
311
+
touch "$DIRTY_FLAG"
312
+
echo "reconcile: pinned missed file $f -> $cid"
313
+
done
314
+
done
272
315
'';
273
316
in
274
317
{