This repository has no description
tools.lazy-store: leave out paths without a listing instead of aborting
B2 behind cache.lassul.us sometimes sends the headers and then no body;
when all retries for one .ls failed, `mount` gave up ("17 of 2892 store
paths without a listing") and the whole build substituted eagerly.
Now a path whose listing can't be fetched (or parsed, or built from its
NAR) is treated like a path not in the caches: it and everything
referring to it (from the references in db.dump) are dropped from
paths.json and db.dump before mounting, so they are neither served nor
registered and Nix substitutes them as usual; the rest stays lazy. Each
is logged once with its error, the counts go to no-listing.json, and
lazy-store.sh prints them in the "mounted" line (and counts the
registered paths after the mount). Narinfo fetch errors already ended up
in `missing` in `closure`.
author
Hermes
committer
lassulus
date
(Oct 17, 2026, 8:08 PM CEST)
commit
64a93820
64a93820ea2be63810979744e5b25c916fba486b
parent
f40e139b
f40e139b04fad5a210487b1259d1fe081b45c389
+150
-22
+7
-3
.github/lazy-store.sh
+7
-3
.github/lazy-store.sh
···
60
60
if [ ! -e "$L/ready" ]; then
61
61
exit 0
62
62
fi
63
-
n=$(jq length "$L/paths.json")
64
63
nixbin=$(dirname "$(readlink -f "$(command -v nix-store)")")
65
64
66
65
sudo modprobe fuse || true
···
77
76
local s
78
77
s=$(ps -o stat= -p "$fuse") && [[ $s != Z* ]]
79
78
}
80
-
# it fetches all .ls listings before mounting
79
+
# it fetches all .ls listings before mounting; paths without one, and those
80
+
# referring to them, it leaves out of paths.json and db.dump, so that the
81
+
# build substitutes them
81
82
until mountpoint -q /nix/store; do
82
83
fuse_alive || fallback "lazy-store exited before mounting /nix/store"
83
84
sleep 1
84
85
done
85
86
grep -m1 '^init: ' "$L/fuse.log" || true
86
87
88
+
n=$(jq length "$L/paths.json")
87
89
some=$(jq -r 'keys[0]' "$L/paths.json")
88
90
[ -e "/nix/store/$some" ] || fallback "/nix/store/$some is not served"
89
91
# shellcheck disable=SC2024 # db.dump is ours, only nix-store needs root
···
91
93
# registered: the mount has to stay
92
94
trap - ERR
93
95
set +e
94
-
echo "lazy store: mounted on /nix/store, $n paths registered"
96
+
grep '^no listing for ' "$L/fuse.log"
97
+
jq -r --arg n "$n" '"lazy store: mounted on /nix/store, \($n) paths registered; \(.failed | length) without a listing, \(.incomplete) referring to those"' \
98
+
"$L/no-listing.json"
95
99
# niks3 and hestia leave out what cache.nixos.org signed by the signatures
96
100
# in nix's database; without them they would upload (and so download)
97
101
# those paths
+17
tools/lazy-store/package.nix
+17
tools/lazy-store/package.nix
···
125
125
nix store copy-sigs --store "$root" -s "file://$PWD/closure/narinfo" --stdin
126
126
nix path-info --store "$root" --sigs /nix/store/$A | grep -q "[[:space:]]test-1:$(printf %64s | base64 -w0)$"
127
127
test "$(nix-store --store "$root" -q --references /nix/store/$A | wc -l)" = 2
128
+
129
+
# b's listing is broken: b and a, which refers to it, are left out,
130
+
# c is served
131
+
echo broken > cache/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.ls
132
+
printf 'StorePath: /nix/store/%s\nURL: nar/c.nar.zst\nCompression: zstd\nNarHash: sha256:%s\nNarSize: %s\nReferences: \n' \
133
+
$C "$(nix-hash --type sha256 --base32 src/c)" "$(size src/c)" > cache/cccccccccccccccccccccccccccccccc.narinfo
134
+
mkdir partial
135
+
printf '/nix/store/%s\n' $A $C | LAZY_STORE_CACHES="file://$PWD/cache" lazy-store closure partial
136
+
lazy-store fetch partial $C
137
+
diff -r src/c partial/cache/$C
138
+
jq -e '.failed == ["'$B'"] and .incomplete == 1' partial/no-listing.json
139
+
jq -e 'keys == ["'$C'"]' partial/paths.json
140
+
test "$(grep -c ^/nix/store/ partial/db.dump)" = 1
141
+
grep -qx /nix/store/$C partial/db.dump
142
+
if lazy-store fetch partial $A; then
143
+
echo "path referring to one without a listing was served"; exit 1
144
+
fi
128
145
touch $out
129
146
'';
130
147
+123
-19
tools/lazy-store/src/lazy.rs
+123
-19
tools/lazy-store/src/lazy.rs
···
4
4
//! (NarHash/NarSize) and unpacks that path's NAR. Opens of a path that is
5
5
//! being fetched wait for that fetch.
6
6
7
-
use std::collections::{BTreeMap, HashMap};
7
+
use std::collections::{BTreeMap, BTreeSet, HashMap};
8
8
use std::ffi::{CStr, CString};
9
9
use std::fs::File;
10
10
use std::io;
···
29
29
use crate::http::{self, Http};
30
30
use crate::nar::{self, Size, Tree};
31
31
use crate::stats::Stats;
32
-
use crate::Backing;
32
+
use crate::{Backing, STORE};
33
33
34
34
/// The store paths never change: let the kernel cache everything.
35
35
pub const TTL: Duration = Duration::from_secs(86400);
···
87
87
io::Error::from_raw_os_error(e)
88
88
}
89
89
90
+
/// `dump` (a `nix-store --load-db` input) without the store paths `failed`
91
+
/// and everything referring to them, like `closure` leaves out what refers
92
+
/// to paths not in the caches; and the names of the paths left out.
93
+
fn leave_out<'a>(
94
+
dump: &'a str,
95
+
failed: impl IntoIterator<Item = &'a str>,
96
+
) -> io::Result<(String, BTreeSet<String>)> {
97
+
let bad = || io::Error::other("malformed db.dump");
98
+
let name = |l: &'a str| l.strip_prefix(STORE).ok_or_else(bad);
99
+
// per path: name, references and its lines
100
+
let mut records = Vec::new();
101
+
let mut lines = dump.split_inclusive('\n');
102
+
while let Some(p) = lines.next() {
103
+
let mut text = p.to_string();
104
+
for _ in 0..3 {
105
+
text += lines.next().ok_or_else(bad)?;
106
+
}
107
+
let count = lines.next().ok_or_else(bad)?;
108
+
text += count;
109
+
let count: usize = count.trim_end().parse().map_err(|_| bad())?;
110
+
let mut refs = Vec::with_capacity(count);
111
+
for _ in 0..count {
112
+
let r = lines.next().ok_or_else(bad)?;
113
+
text += r;
114
+
refs.push(name(r.trim_end())?);
115
+
}
116
+
records.push((name(p.trim_end())?, refs, text));
117
+
}
118
+
let mut referrers: HashMap<&str, Vec<&str>> = HashMap::new();
119
+
for (n, refs, _) in &records {
120
+
for r in refs.iter().filter(|r| *r != n) {
121
+
referrers.entry(r).or_default().push(n);
122
+
}
123
+
}
124
+
let mut out: BTreeSet<&str> = failed.into_iter().collect();
125
+
let mut todo: Vec<&str> = out.iter().copied().collect();
126
+
while let Some(m) = todo.pop() {
127
+
for r in referrers.get(m).into_iter().flatten() {
128
+
if out.insert(r) {
129
+
todo.push(r);
130
+
}
131
+
}
132
+
}
133
+
let kept = records
134
+
.iter()
135
+
.filter(|(n, _, _)| !out.contains(n))
136
+
.map(|(_, _, t)| t.as_str())
137
+
.collect();
138
+
Ok((kept, out.into_iter().map(str::to_string).collect()))
139
+
}
140
+
90
141
impl LazyFs {
91
142
pub async fn new(outdir: &Path, backing: Option<Arc<Backing>>) -> io::Result<LazyFs> {
92
143
let paths: BTreeMap<String, PathInfo> =
···
111
162
backing_ids: Mutex::new(HashMap::new()),
112
163
backing_failed: AtomicBool::new(false),
113
164
};
114
-
fs.load_listings().await?;
165
+
fs.load_listings(outdir).await?;
115
166
fs.stats.save();
116
167
Ok(fs)
117
168
}
118
169
119
170
/// Fetch the `.ls` listing of every store path (falling back to the NAR
120
-
/// for paths without one) and build the inode table from them. Fails if
121
-
/// any path is left without a listing: all of them get registered as
122
-
/// valid in nix's database.
123
-
async fn load_listings(&mut self) -> io::Result<()> {
171
+
/// for paths without one) and build the inode table from them. Paths
172
+
/// left without a listing, and everything referring to them, are left
173
+
/// out of OUT/paths.json and OUT/db.dump (see `leave_out`), so they are
174
+
/// neither served nor registered; OUT/no-listing.json counts them.
175
+
async fn load_listings(&mut self, outdir: &Path) -> io::Result<()> {
124
176
let t = Instant::now();
125
177
let total = self.names.len();
126
178
let mut trees: Vec<Option<io::Result<Tree>>> = (0..total).map(|_| None).collect();
···
144
196
path: u32::MAX,
145
197
rel: "".into(),
146
198
});
147
-
let mut failed = 0;
199
+
let mut ok = Vec::with_capacity(total);
200
+
let mut failed = Vec::new();
148
201
for (i, t) in trees.enumerate() {
149
202
match t {
150
-
Ok(t) => {
151
-
let ino = self.add(i as u32, String::new(), t);
152
-
root.push((self.names[i].clone().into_boxed_str(), ino));
153
-
}
203
+
Ok(t) => ok.push((i, t)),
154
204
Err(e) => {
155
205
eprintln!("no listing for {}: {e}", self.names[i]);
156
-
failed += 1;
206
+
failed.push(self.names[i].clone());
157
207
}
158
208
}
159
209
}
160
-
if failed > 0 {
161
-
return Err(io::Error::other(format!(
162
-
"{failed} of {} store paths without a listing",
163
-
self.names.len()
164
-
)));
210
+
let mut left_out = BTreeSet::new();
211
+
if !failed.is_empty() {
212
+
let dump = std::fs::read_to_string(outdir.join("db.dump"))?;
213
+
let (kept, out) = leave_out(&dump, failed.iter().map(String::as_str))?;
214
+
std::fs::write(outdir.join("db.dump"), kept)?;
215
+
let paths: BTreeMap<&String, &PathInfo> = self
216
+
.names
217
+
.iter()
218
+
.zip(&self.infos)
219
+
.filter(|(n, _)| !out.contains(n.as_str()))
220
+
.collect();
221
+
std::fs::write(outdir.join("paths.json"), serde_json::to_vec(&paths)?)?;
222
+
left_out = out;
223
+
}
224
+
std::fs::write(
225
+
outdir.join("no-listing.json"),
226
+
serde_json::to_vec(&serde_json::json!({
227
+
"failed": failed,
228
+
"incomplete": left_out.len() - failed.len(),
229
+
}))?,
230
+
)?;
231
+
for (i, t) in ok {
232
+
if left_out.contains(&self.names[i]) {
233
+
continue;
234
+
}
235
+
let ino = self.add(i as u32, String::new(), t);
236
+
root.push((self.names[i].clone().into_boxed_str(), ino));
165
237
}
166
238
root.sort();
167
239
self.nodes[0].kind = Kind::Dir(root);
168
240
eprintln!(
169
-
"listings: {} store paths, {} inodes in {:.1} s",
241
+
"listings: {} store paths, {} inodes in {:.1} s; {} without a listing, {} referring to those left out",
170
242
self.names.len(),
171
243
self.nodes.len(),
172
244
t.elapsed().as_secs_f64(),
245
+
failed.len(),
246
+
left_out.len() - failed.len(),
173
247
);
174
248
Ok(())
175
249
}
···
634
708
return Err(err(libc::EROFS));
635
709
}
636
710
Ok(())
711
+
}
712
+
}
713
+
714
+
#[cfg(test)]
715
+
mod tests {
716
+
use super::*;
717
+
718
+
#[test]
719
+
fn leave_out_referrers() {
720
+
let rec = |n: &str, refs: &[&str]| {
721
+
let mut s = format!("{STORE}{n}\nsha256:x\n8\n\n{}\n", refs.len());
722
+
for r in refs {
723
+
s += &format!("{STORE}{r}\n");
724
+
}
725
+
s
726
+
};
727
+
// w -> x -> y (failed), y -> y, z
728
+
let dump = [
729
+
rec("w", &["x"]),
730
+
rec("x", &["x", "y"]),
731
+
rec("y", &["y"]),
732
+
rec("z", &[]),
733
+
]
734
+
.concat();
735
+
let (kept, out) = leave_out(&dump, ["y"]).unwrap();
736
+
assert_eq!(kept, rec("z", &[]));
737
+
assert_eq!(out, ["w", "x", "y"].map(String::from).into());
738
+
let (kept, out) = leave_out(&dump, []).unwrap();
739
+
assert_eq!(kept, dump);
740
+
assert!(out.is_empty());
637
741
}
638
742
}
+3
tools/lazy-store/src/main.rs
+3
tools/lazy-store/src/main.rs
···
12
12
//! their listing (testing without FUSE)
13
13
//! mount OUT UPPER MNT mount the store dir UPPER, merged with the paths
14
14
//! of OUT/paths.json, at MNT (see storefs.rs).
15
+
//! Paths without a listing, and those referring
16
+
//! to them, are left out of OUT/paths.json and
17
+
//! OUT/db.dump (counted in OUT/no-listing.json).
15
18
//! Stats go to OUT/stats.json.
16
19
//! [--workers N] FUSE worker threads (default 64)
17
20
//! [--no-passthrough] don't use kernel FUSE passthrough