This repository has no description
tools.lazy-store: use harmonia for nixbase32 and the .ls listing format
Replace our nix32 encoder with harmonia-utils-base-encoding's base32 and
the hand-walked serde_json::Value of the caches' .ls listings with
harmonia-file-core's FileTree, which is the serde model of Nix's file
system object JSON. Both are pure crates (data-encoding/derive_more,
serde/serde_json/thiserror), pulled as a git dependency pinned to
harmonia 3edb37ae: only harmonia-utils-hash and harmonia-store-core exist
on crates.io, as 0.0.0-alpha.0 placeholders. A listing node without
"size" (regular) or "entries" (directory) is now an error instead of
being read as 0 / empty; Nix always writes both.
Kept ours:
- NAR unpacking: harmonia-file-nar is a tokio AsyncRead parser/restorer;
we unpack synchronously from the blocking HTTP/decompressor stream
while hashing, in a sync FUSE daemon.
- narinfo parsing, store path names and NarHash parsing:
harmonia-store-nar-info, -store-path and -utils-hash all pull in
aws-lc-rs (aws-lc-sys C build), blake3 and md5, nar-info also
ed25519-dalek and getrandom: +43 crates for ~15 lines. closure also
passes the narinfo text through verbatim, and harmonia's parser
rejects whole narinfos for fields we don't use (Sig, CA, FileHash).
- The in-memory Tree: matching FileTree(FileSystemObject::Regular(
Regular { .. })) in lazy.rs is clumsier than our flat enum.
author
Hermes
committer
lassulus
date
(Oct 17, 2026, 7:03 PM CEST)
commit
207a89b5
207a89b55362329890ec3bdd21b6a3e7b900af41
parent
ffdcad2c
ffdcad2cb8522d5aa40bfe79daac02f5893eb61e
+151
-54
+105
-7
tools/lazy-store/Cargo.lock
+105
-7
tools/lazy-store/Cargo.lock
···
117
117
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
118
118
119
119
[[package]]
120
+
name = "convert_case"
121
+
version = "0.10.0"
122
+
source = "registry+https://github.com/rust-lang/crates.io-index"
123
+
checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9"
124
+
dependencies = [
125
+
"unicode-segmentation",
126
+
]
127
+
128
+
[[package]]
120
129
name = "core-foundation-sys"
121
130
version = "0.8.7"
122
131
source = "registry+https://github.com/rust-lang/crates.io-index"
···
151
160
]
152
161
153
162
[[package]]
163
+
name = "data-encoding"
164
+
version = "2.11.1"
165
+
source = "registry+https://github.com/rust-lang/crates.io-index"
166
+
checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06"
167
+
168
+
[[package]]
169
+
name = "derive_more"
170
+
version = "2.1.1"
171
+
source = "registry+https://github.com/rust-lang/crates.io-index"
172
+
checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134"
173
+
dependencies = [
174
+
"derive_more-impl",
175
+
]
176
+
177
+
[[package]]
178
+
name = "derive_more-impl"
179
+
version = "2.1.1"
180
+
source = "registry+https://github.com/rust-lang/crates.io-index"
181
+
checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb"
182
+
dependencies = [
183
+
"convert_case",
184
+
"proc-macro2",
185
+
"quote",
186
+
"rustc_version",
187
+
"syn 2.0.119",
188
+
"unicode-xid",
189
+
]
190
+
191
+
[[package]]
154
192
name = "digest"
155
193
version = "0.10.7"
156
194
source = "registry+https://github.com/rust-lang/crates.io-index"
···
168
206
dependencies = [
169
207
"proc-macro2",
170
208
"quote",
171
-
"syn",
209
+
"syn 3.0.9",
172
210
]
173
211
174
212
[[package]]
···
311
349
]
312
350
313
351
[[package]]
352
+
name = "harmonia-file-core"
353
+
version = "3.3.0"
354
+
source = "git+https://github.com/nix-community/harmonia?rev=3edb37ae36b561c2d395980170c70bbce292bbc5#3edb37ae36b561c2d395980170c70bbce292bbc5"
355
+
dependencies = [
356
+
"serde",
357
+
"serde_json",
358
+
"thiserror",
359
+
]
360
+
361
+
[[package]]
362
+
name = "harmonia-utils-base-encoding"
363
+
version = "0.0.0-alpha.0"
364
+
source = "git+https://github.com/nix-community/harmonia?rev=3edb37ae36b561c2d395980170c70bbce292bbc5#3edb37ae36b561c2d395980170c70bbce292bbc5"
365
+
dependencies = [
366
+
"data-encoding",
367
+
"derive_more",
368
+
"serde",
369
+
]
370
+
371
+
[[package]]
314
372
name = "icu_collections"
315
373
version = "2.3.0"
316
374
source = "registry+https://github.com/rust-lang/crates.io-index"
···
438
496
"bzip2",
439
497
"flate2",
440
498
"fuse-backend-rs",
499
+
"harmonia-file-core",
500
+
"harmonia-utils-base-encoding",
441
501
"libc",
442
502
"log",
443
503
"serde",
···
611
671
]
612
672
613
673
[[package]]
674
+
name = "rustc_version"
675
+
version = "0.4.1"
676
+
source = "registry+https://github.com/rust-lang/crates.io-index"
677
+
checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
678
+
dependencies = [
679
+
"semver",
680
+
]
681
+
682
+
[[package]]
614
683
name = "rustls"
615
684
version = "0.23.46"
616
685
source = "registry+https://github.com/rust-lang/crates.io-index"
···
652
721
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
653
722
654
723
[[package]]
724
+
name = "semver"
725
+
version = "1.0.28"
726
+
source = "registry+https://github.com/rust-lang/crates.io-index"
727
+
checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
728
+
729
+
[[package]]
655
730
name = "serde"
656
731
version = "1.0.229"
657
732
source = "registry+https://github.com/rust-lang/crates.io-index"
···
678
753
dependencies = [
679
754
"proc-macro2",
680
755
"quote",
681
-
"syn",
756
+
"syn 3.0.9",
682
757
]
683
758
684
759
[[package]]
···
737
812
738
813
[[package]]
739
814
name = "syn"
815
+
version = "2.0.119"
816
+
source = "registry+https://github.com/rust-lang/crates.io-index"
817
+
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
818
+
dependencies = [
819
+
"proc-macro2",
820
+
"quote",
821
+
"unicode-ident",
822
+
]
823
+
824
+
[[package]]
825
+
name = "syn"
740
826
version = "3.0.9"
741
827
source = "registry+https://github.com/rust-lang/crates.io-index"
742
828
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
···
754
840
dependencies = [
755
841
"proc-macro2",
756
842
"quote",
757
-
"syn",
843
+
"syn 3.0.9",
758
844
]
759
845
760
846
[[package]]
···
774
860
dependencies = [
775
861
"proc-macro2",
776
862
"quote",
777
-
"syn",
863
+
"syn 3.0.9",
778
864
]
779
865
780
866
[[package]]
···
798
884
version = "1.0.27"
799
885
source = "registry+https://github.com/rust-lang/crates.io-index"
800
886
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
887
+
888
+
[[package]]
889
+
name = "unicode-segmentation"
890
+
version = "1.13.3"
891
+
source = "registry+https://github.com/rust-lang/crates.io-index"
892
+
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
893
+
894
+
[[package]]
895
+
name = "unicode-xid"
896
+
version = "0.2.6"
897
+
source = "registry+https://github.com/rust-lang/crates.io-index"
898
+
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
801
899
802
900
[[package]]
803
901
name = "untrusted"
···
1084
1182
dependencies = [
1085
1183
"proc-macro2",
1086
1184
"quote",
1087
-
"syn",
1185
+
"syn 3.0.9",
1088
1186
"synstructure",
1089
1187
]
1090
1188
···
1105
1203
dependencies = [
1106
1204
"proc-macro2",
1107
1205
"quote",
1108
-
"syn",
1206
+
"syn 3.0.9",
1109
1207
"synstructure",
1110
1208
]
1111
1209
···
1145
1243
dependencies = [
1146
1244
"proc-macro2",
1147
1245
"quote",
1148
-
"syn",
1246
+
"syn 3.0.9",
1149
1247
]
1150
1248
1151
1249
[[package]]
+2
tools/lazy-store/Cargo.toml
+2
tools/lazy-store/Cargo.toml
···
21
21
bzip2 = { version = "0.4", features = ["static"] }
22
22
flate2 = "1"
23
23
brotli-decompressor = "4"
24
+
harmonia-file-core = { git = "https://github.com/nix-community/harmonia", rev = "3edb37ae36b561c2d395980170c70bbce292bbc5" }
25
+
harmonia-utils-base-encoding = { git = "https://github.com/nix-community/harmonia", rev = "3edb37ae36b561c2d395980170c70bbce292bbc5" }
24
26
25
27
[profile.release]
26
28
debug = 1
+1
tools/lazy-store/package.nix
+1
tools/lazy-store/package.nix
···
22
22
cargoLock = {
23
23
lockFile = ./Cargo.lock;
24
24
outputHashes."fuse-backend-rs-0.14.0" = "sha256-pSnYq8IhfpxtnTS7bnE8GHAVP+yZ+hItChTEmGUP7xM=";
25
+
outputHashes."harmonia-file-core-3.3.0" = "sha256-aPNxxJLjRd+wVFyAcUnzrakC0BNj/N9rJWV2y8wpCAs=";
25
26
};
26
27
# Static, so that once it serves /nix/store it loads nothing from there.
27
28
# (Not glibc.static in buildInputs: build scripts would link against it.)
+2
-1
tools/lazy-store/src/lazy.rs
+2
-1
tools/lazy-store/src/lazy.rs
···
20
20
Context, DirEntry, Entry, FileSystem, GetxattrReply, ListxattrReply, OpenOptions,
21
21
ZeroCopyWriter,
22
22
};
23
+
use harmonia_utils_base_encoding::base32;
23
24
24
25
use crate::closure::{par_map, PathInfo};
25
26
use crate::http::{self, Http};
···
345
346
nar::remove(&tmp)?;
346
347
return Err(io::Error::other(format!(
347
348
"{name}: NAR hash mismatch, sha256:{} ({size} bytes) != {} ({} bytes)",
348
-
nar::nix32(&digest),
349
+
base32::encode_string(&digest),
349
350
info.nar_hash,
350
351
info.nar_size
351
352
)));
+41
-46
tools/lazy-store/src/nar.rs
+41
-46
tools/lazy-store/src/nar.rs
···
7
7
use std::os::unix::fs::{symlink, MetadataExt, PermissionsExt};
8
8
use std::path::Path;
9
9
10
+
use harmonia_file_core::{FileSystemObject, FileTree};
11
+
use harmonia_utils_base_encoding::base32;
12
+
use serde::Deserialize;
10
13
use sha2::{Digest, Sha256};
11
-
12
-
const NIX32: &[u8] = b"0123456789abcdfghijklmnpqrsvwxyz";
13
-
14
-
pub fn nix32(b: &[u8]) -> String {
15
-
let n = (b.len() * 8 - 1) / 5 + 1;
16
-
(0..n)
17
-
.rev()
18
-
.map(|i| {
19
-
let bit = i * 5;
20
-
let (j, k) = (bit / 8, bit % 8);
21
-
let mut c = (b[j] as u32) >> k;
22
-
if j + 1 < b.len() {
23
-
c |= (b[j + 1] as u32) << (8 - k);
24
-
}
25
-
NIX32[(c & 0x1f) as usize] as char
26
-
})
27
-
.collect()
28
-
}
29
14
30
15
/// Does `digest` match a narinfo NarHash (`sha256:<nix32>` or base16)?
31
16
pub fn hash_matches(expected: &str, digest: &[u8]) -> bool {
···
33
18
return false;
34
19
};
35
20
let hex: String = digest.iter().map(|b| format!("{b:02x}")).collect();
36
-
h == nix32(digest) || h == hex
21
+
h == base32::encode_string(digest) || h == hex
37
22
}
38
23
39
24
/// The metadata of a store path, as in the caches' `.ls` listings.
···
47
32
impl Tree {
48
33
/// Parse a `.ls` listing (`{"version": 1, "root": {...}}`).
49
34
pub fn from_listing(body: &[u8]) -> io::Result<Tree> {
50
-
let v: serde_json::Value = serde_json::from_slice(body).map_err(io::Error::other)?;
51
-
Tree::from_json(&v["root"])
52
-
}
53
-
54
-
fn from_json(v: &serde_json::Value) -> io::Result<Tree> {
55
-
let bad = || io::Error::other(format!("bad listing node {v}"));
56
-
Ok(match v["type"].as_str().ok_or_else(bad)? {
57
-
"directory" => {
58
-
let mut m = BTreeMap::new();
59
-
if let Some(e) = v["entries"].as_object() {
60
-
for (k, c) in e {
61
-
m.insert(k.clone(), Tree::from_json(c)?);
62
-
}
63
-
}
64
-
Tree::Dir(m)
35
+
#[derive(Deserialize)]
36
+
struct Size {
37
+
size: u64,
38
+
}
39
+
#[derive(Deserialize)]
40
+
struct Listing {
41
+
root: FileTree<Size>,
42
+
}
43
+
fn tree(FileTree(t): FileTree<Size>) -> Tree {
44
+
match t {
45
+
FileSystemObject::Directory(d) => Tree::Dir(
46
+
d.entries
47
+
.into_iter()
48
+
.map(|(k, c)| (k, tree(*c)))
49
+
.collect(),
50
+
),
51
+
FileSystemObject::Regular(r) => Tree::File {
52
+
size: r.contents.size,
53
+
exec: r.executable,
54
+
},
55
+
FileSystemObject::Symlink(s) => Tree::Symlink(s.target),
65
56
}
66
-
"regular" => Tree::File {
67
-
size: v["size"].as_u64().unwrap_or(0),
68
-
exec: v["executable"].as_bool().unwrap_or(false),
69
-
},
70
-
"symlink" => Tree::Symlink(v["target"].as_str().ok_or_else(bad)?.to_string()),
71
-
_ => return Err(bad()),
72
-
})
57
+
}
58
+
let l: Listing = serde_json::from_slice(body).map_err(io::Error::other)?;
59
+
Ok(tree(l.root))
73
60
}
74
61
75
62
/// Listing of an unpacked store path (for paths without `.ls`).
···
329
316
}
330
317
331
318
#[test]
332
-
fn nix32_matches_nix() {
319
+
fn hash_matches_nix() {
333
320
// nix hash convert --to nix32 sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=
334
321
let d = Sha256::digest(b"");
335
-
assert_eq!(
336
-
nix32(&d),
337
-
"0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73"
338
-
);
322
+
assert!(hash_matches(
323
+
"sha256:0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73",
324
+
&d
325
+
));
326
+
assert!(hash_matches(
327
+
"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
328
+
&d
329
+
));
330
+
assert!(!hash_matches(
331
+
"0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73",
332
+
&d
333
+
));
339
334
}
340
335
341
336
#[test]