This repository has no description
flix/ipfs: drop pin-add lock (concurrent --nocopy is idempotent)
The flock around `ipfs add --nocopy` in the watcher and reconcile sweep
guarded against a corruption that was assumed, never observed: it was
added defensively alongside the reconcile sweep and never exercised
(the MFS deadlock masked it).
Tested it directly on kubo 0.40.1: 10 trials of two concurrent
`ipfs add --nocopy --pin` on the same 16MB file, no lock. Every trial
produced the same CID, a complete DAG (refs -r), and byte-identical
content (cat | sha256sum). Concurrent same-file --nocopy adds are
idempotent, so the lock guards nothing -- remove it.
author
lassulus
date
(Jun 26, 2026, 1:49 PM CEST)
commit
3bd60d2f
3bd60d2ffc430230834672b6e5374276560542dd
parent
6a69b329
6a69b329c1ba76ecfdccfe5439843ab36936d885
+6
-12
+6
-12
2configs/services/flix/ipfs.nix
+6
-12
2configs/services/flix/ipfs.nix
···
12
12
];
13
13
14
14
cidMapFile = "/var/lib/ipfs/cid-map.txt";
15
-
# Serializes `ipfs add --nocopy` between the pin-watcher and the reconcile
16
-
# sweep: two concurrent --nocopy adds of the same file corrupt the filestore
17
-
# into an incomplete DAG.
18
-
pinAddLock = "/var/lib/ipfs/pin-add.lock";
19
15
20
16
pinWatcherScript = pkgs.writers.writeBash "ipfs-pin-watcher" ''
21
17
set -efu
···
23
19
IPFS="${pkgs.kubo}/bin/ipfs"
24
20
INOTIFYWAIT="${pkgs.inotify-tools}/bin/inotifywait"
25
21
CID_MAP="${cidMapFile}"
26
-
PIN_LOCK="${pinAddLock}"
27
22
28
23
touch "$CID_MAP"
29
24
${pkgs.coreutils}/bin/chmod 0644 "$CID_MAP"
···
40
35
esac
41
36
42
37
log "Adding $path"
43
-
cid=$(${pkgs.util-linux}/bin/flock "$PIN_LOCK" \
44
-
$IPFS add --nocopy --pin --quieter "$path" 2>/dev/null) || {
38
+
cid=$($IPFS add --nocopy --pin --quieter "$path" 2>/dev/null) || {
45
39
log "Failed to add $path"
46
40
return 0
47
41
}
···
139
133
140
134
IPFS="${pkgs.kubo}/bin/ipfs"
141
135
CID_MAP="${cidMapFile}"
142
-
PIN_LOCK="${pinAddLock}"
143
136
144
137
[ -f "$CID_MAP" ] || exit 0
145
138
···
168
161
fi
169
162
170
163
# Forward: pin on-disk files missing from cid-map. Only files older than
171
-
# 10 min, so the flock + margin never collide with the watcher mid-add on a
172
-
# fresh upload (a file that old and still absent was dropped, not in-flight).
164
+
# 10 min -- a file that old and still absent from cid-map was dropped by the
165
+
# watcher, not in-flight. No lock against the watcher: concurrent
166
+
# `add --nocopy` of the same file is idempotent (verified), so a rare
167
+
# double-add is harmless.
173
168
for dir in ${lib.escapeShellArgs watchDirs}; do
174
169
[ -d "$dir" ] || continue
175
170
${pkgs.findutils}/bin/find "$dir" -type f -mmin +10 | while read -r f; do
···
179
174
if ${pkgs.gnugrep}/bin/grep -q " $f$" "$CID_MAP" 2>/dev/null; then
180
175
continue
181
176
fi
182
-
cid=$(${pkgs.util-linux}/bin/flock "$PIN_LOCK" \
183
-
$IPFS add --nocopy --pin --quieter "$f" 2>/dev/null) || {
177
+
cid=$($IPFS add --nocopy --pin --quieter "$f" 2>/dev/null) || {
184
178
echo "reconcile: failed to add $f"
185
179
continue
186
180
}