This repository has no description
0

Configure Feed

Select the types of activity you want to include in your feed.

tools.lazy-store: parse narinfos and store paths with harmonia

closure uses harmonia-store-nar-info's parse_narinfo_txt and
harmonia-store-path's StorePath instead of our key/value narinfo map
and string splitting: the closure walk, referrers and missing sets are
keyed by StorePath (whose order is the string order, so paths.json and
db.dump come out the same), references, deriver, NarHash, NarSize,
URL, Compression and FileSize come from the parsed NarInfo. A narinfo
that doesn't parse counts as a failed fetch from that cache (try the
next one) instead of failing the whole run later on a missing field.
Roots that aren't store paths are an error.

paths.json, db.dump, narinfo/ and closure.json are byte-identical to
before for the closures of hello and git from cache.nixos.org.

The fetch test's path without a narinfo used an "e" hash, which isn't
nix32 and so no store path.

+243 -91
+153 -4
tools/lazy-store/Cargo.lock
··· 329 329 ] 330 330 331 331 [[package]] 332 + name = "curve25519-dalek" 333 + version = "5.0.0" 334 + source = "registry+https://github.com/rust-lang/crates.io-index" 335 + checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" 336 + dependencies = [ 337 + "cfg-if", 338 + "cpufeatures", 339 + "curve25519-dalek-derive", 340 + "digest", 341 + "fiat-crypto", 342 + "rustc_version", 343 + "subtle", 344 + "zeroize", 345 + ] 346 + 347 + [[package]] 348 + name = "curve25519-dalek-derive" 349 + version = "0.1.1" 350 + source = "registry+https://github.com/rust-lang/crates.io-index" 351 + checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" 352 + dependencies = [ 353 + "proc-macro2", 354 + "quote", 355 + "syn 2.0.119", 356 + ] 357 + 358 + [[package]] 332 359 name = "data-encoding" 333 360 version = "2.11.1" 334 361 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 386 413 checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" 387 414 388 415 [[package]] 416 + name = "ed25519" 417 + version = "3.0.0" 418 + source = "registry+https://github.com/rust-lang/crates.io-index" 419 + checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" 420 + dependencies = [ 421 + "signature", 422 + ] 423 + 424 + [[package]] 425 + name = "ed25519-dalek" 426 + version = "3.0.0" 427 + source = "registry+https://github.com/rust-lang/crates.io-index" 428 + checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" 429 + dependencies = [ 430 + "curve25519-dalek", 431 + "ed25519", 432 + "sha2", 433 + "subtle", 434 + "zeroize", 435 + ] 436 + 437 + [[package]] 389 438 name = "endian-type" 390 439 version = "0.1.2" 391 440 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 404 453 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" 405 454 dependencies = [ 406 455 "libc", 407 - "windows-sys 0.52.0", 456 + "windows-sys 0.61.2", 408 457 ] 458 + 459 + [[package]] 460 + name = "fiat-crypto" 461 + version = "0.3.0" 462 + source = "registry+https://github.com/rust-lang/crates.io-index" 463 + checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" 409 464 410 465 [[package]] 411 466 name = "find-msvc-tools" ··· 661 716 ] 662 717 663 718 [[package]] 719 + name = "harmonia-store-content-address" 720 + version = "3.3.0" 721 + source = "git+https://github.com/nix-community/harmonia?rev=3edb37ae36b561c2d395980170c70bbce292bbc5#3edb37ae36b561c2d395980170c70bbce292bbc5" 722 + dependencies = [ 723 + "derive_more", 724 + "harmonia-store-path", 725 + "harmonia-utils-hash", 726 + "serde", 727 + "thiserror", 728 + ] 729 + 730 + [[package]] 731 + name = "harmonia-store-nar-info" 732 + version = "3.3.0" 733 + source = "git+https://github.com/nix-community/harmonia?rev=3edb37ae36b561c2d395980170c70bbce292bbc5#3edb37ae36b561c2d395980170c70bbce292bbc5" 734 + dependencies = [ 735 + "harmonia-store-content-address", 736 + "harmonia-store-path", 737 + "harmonia-store-path-info", 738 + "harmonia-utils-hash", 739 + "harmonia-utils-signature", 740 + "serde", 741 + "thiserror", 742 + ] 743 + 744 + [[package]] 745 + name = "harmonia-store-path" 746 + version = "3.3.0" 747 + source = "git+https://github.com/nix-community/harmonia?rev=3edb37ae36b561c2d395980170c70bbce292bbc5#3edb37ae36b561c2d395980170c70bbce292bbc5" 748 + dependencies = [ 749 + "derive_more", 750 + "harmonia-utils-base-encoding", 751 + "harmonia-utils-hash", 752 + "serde", 753 + "thiserror", 754 + "zerocopy", 755 + ] 756 + 757 + [[package]] 758 + name = "harmonia-store-path-info" 759 + version = "3.3.0" 760 + source = "git+https://github.com/nix-community/harmonia?rev=3edb37ae36b561c2d395980170c70bbce292bbc5#3edb37ae36b561c2d395980170c70bbce292bbc5" 761 + dependencies = [ 762 + "harmonia-store-content-address", 763 + "harmonia-store-path", 764 + "harmonia-utils-hash", 765 + "harmonia-utils-signature", 766 + "serde", 767 + ] 768 + 769 + [[package]] 664 770 name = "harmonia-utils-base-encoding" 665 771 version = "0.0.0-alpha.0" 666 772 source = "git+https://github.com/nix-community/harmonia?rev=3edb37ae36b561c2d395980170c70bbce292bbc5#3edb37ae36b561c2d395980170c70bbce292bbc5" ··· 700 806 "nix 0.31.3", 701 807 "pin-project-lite", 702 808 "tokio", 809 + ] 810 + 811 + [[package]] 812 + name = "harmonia-utils-signature" 813 + version = "3.3.0" 814 + source = "git+https://github.com/nix-community/harmonia?rev=3edb37ae36b561c2d395980170c70bbce292bbc5#3edb37ae36b561c2d395980170c70bbce292bbc5" 815 + dependencies = [ 816 + "data-encoding", 817 + "ed25519-dalek", 818 + "getrandom 0.4.3", 819 + "harmonia-utils-base-encoding", 820 + "serde", 821 + "subtle", 822 + "thiserror", 823 + "zeroize", 703 824 ] 704 825 705 826 [[package]] ··· 1021 1142 "futures-util", 1022 1143 "harmonia-file-core", 1023 1144 "harmonia-file-nar", 1145 + "harmonia-store-nar-info", 1146 + "harmonia-store-path", 1024 1147 "harmonia-utils-hash", 1025 1148 "libc", 1026 1149 "log", ··· 1281 1404 "once_cell", 1282 1405 "socket2", 1283 1406 "tracing", 1284 - "windows-sys 0.52.0", 1407 + "windows-sys 0.61.2", 1285 1408 ] 1286 1409 1287 1410 [[package]] ··· 1463 1586 "security-framework", 1464 1587 "security-framework-sys", 1465 1588 "webpki-root-certs", 1466 - "windows-sys 0.52.0", 1589 + "windows-sys 0.61.2", 1467 1590 ] 1468 1591 1469 1592 [[package]] ··· 1617 1740 "errno", 1618 1741 "libc", 1619 1742 ] 1743 + 1744 + [[package]] 1745 + name = "signature" 1746 + version = "3.0.0" 1747 + source = "registry+https://github.com/rust-lang/crates.io-index" 1748 + checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" 1620 1749 1621 1750 [[package]] 1622 1751 name = "simd-adler32" ··· 2105 2234 source = "registry+https://github.com/rust-lang/crates.io-index" 2106 2235 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" 2107 2236 dependencies = [ 2108 - "windows-sys 0.52.0", 2237 + "windows-sys 0.48.0", 2109 2238 ] 2110 2239 2111 2240 [[package]] ··· 2295 2424 "quote", 2296 2425 "syn 3.0.9", 2297 2426 "synstructure", 2427 + ] 2428 + 2429 + [[package]] 2430 + name = "zerocopy" 2431 + version = "0.8.63" 2432 + source = "registry+https://github.com/rust-lang/crates.io-index" 2433 + checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" 2434 + dependencies = [ 2435 + "zerocopy-derive", 2436 + ] 2437 + 2438 + [[package]] 2439 + name = "zerocopy-derive" 2440 + version = "0.8.63" 2441 + source = "registry+https://github.com/rust-lang/crates.io-index" 2442 + checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" 2443 + dependencies = [ 2444 + "proc-macro2", 2445 + "quote", 2446 + "syn 2.0.119", 2298 2447 ] 2299 2448 2300 2449 [[package]]
+2
tools/lazy-store/Cargo.toml
··· 23 23 async-compression = { version = "0.4", features = ["tokio", "zstd", "xz", "bzip2", "gzip", "brotli"] } 24 24 harmonia-file-core = { git = "https://github.com/nix-community/harmonia", rev = "3edb37ae36b561c2d395980170c70bbce292bbc5" } 25 25 harmonia-file-nar = { git = "https://github.com/nix-community/harmonia", rev = "3edb37ae36b561c2d395980170c70bbce292bbc5" } 26 + harmonia-store-nar-info = { git = "https://github.com/nix-community/harmonia", rev = "3edb37ae36b561c2d395980170c70bbce292bbc5" } 27 + harmonia-store-path = { git = "https://github.com/nix-community/harmonia", rev = "3edb37ae36b561c2d395980170c70bbce292bbc5" } 26 28 harmonia-utils-hash = { git = "https://github.com/nix-community/harmonia", rev = "3edb37ae36b561c2d395980170c70bbce292bbc5" } 27 29 28 30 [profile.release]
+2 -1
tools/lazy-store/package.nix
··· 63 63 B=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-b 64 64 C=cccccccccccccccccccccccccccccccc-c 65 65 D=dddddddddddddddddddddddddddddddd-d 66 - E=eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-e 66 + # store path hashes are nix32, which has no e 67 + E=ffffffffffffffffffffffffffffffff-f 67 68 ls='{"version":1,"root":{"type":"directory","entries":{ 68 69 "file":{"type":"regular","size":6}, 69 70 "link":{"type":"symlink","target":"file"},
+86 -86
tools/lazy-store/src/closure.rs
··· 6 6 use std::path::Path; 7 7 8 8 use futures_util::{stream, StreamExt}; 9 + use harmonia_store_nar_info::{parse_narinfo_txt, NarInfo}; 10 + use harmonia_store_path::{StoreDir, StorePath}; 11 + use harmonia_utils_hash::{Hash, HashFormat}; 9 12 use serde::{Deserialize, Serialize}; 10 13 11 14 use crate::http::{self, Http}; ··· 32 35 .collect() 33 36 } 34 37 35 - fn parse_narinfo(text: &[u8]) -> HashMap<String, String> { 36 - String::from_utf8_lossy(text) 37 - .lines() 38 - .filter_map(|l| l.split_once(": ")) 39 - .map(|(k, v)| (k.to_string(), v.to_string())) 40 - .collect() 38 + /// A narinfo, the cache it is from and its text. 39 + struct Found { 40 + cache: String, 41 + info: NarInfo, 42 + text: Vec<u8>, 41 43 } 42 44 43 - fn references(i: &HashMap<String, String>) -> impl Iterator<Item = &str> { 44 - i.get("References") 45 - .map(String::as_str) 46 - .unwrap_or("") 47 - .split_whitespace() 45 + impl Found { 46 + fn references(&self) -> &BTreeSet<StorePath> { 47 + &self.info.info.info.references 48 + } 48 49 } 49 50 50 - /// The parsed narinfo of `name` from the first cache that has it, and its 51 - /// text. 52 - async fn narinfo( 53 - http: &Http, 54 - caches: &[String], 55 - name: &str, 56 - stats: &Stats, 57 - ) -> Option<(HashMap<String, String>, Vec<u8>)> { 58 - let h = name.split('-').next().unwrap_or(name); 51 + /// The narinfo of `path` from the first cache that has it. 52 + async fn narinfo(http: &Http, caches: &[String], path: &StorePath, stats: &Stats) -> Option<Found> { 53 + let h = path.hash(); 59 54 for c in caches { 60 - match http.get(&format!("{c}/{h}.narinfo")).await { 61 - Ok((body, len)) => { 55 + let url = format!("{c}/{h}.narinfo"); 56 + let err = match http.get(&url).await { 57 + Ok((text, len)) => { 62 58 stats.add("narinfo", len as u64); 63 - let mut i = parse_narinfo(&body); 64 - i.insert("cache".into(), c.clone()); 65 - return Some((i, body)); 59 + let parsed = std::str::from_utf8(&text) 60 + .map_err(|e| e.to_string()) 61 + .and_then(|s| { 62 + parse_narinfo_txt(&StoreDir::default(), s).map_err(|e| e.to_string()) 63 + }); 64 + match parsed { 65 + Ok(info) => { 66 + let cache = c.clone(); 67 + return Some(Found { cache, info, text }); 68 + } 69 + Err(e) => e, 70 + } 66 71 } 67 - Err(http::Error::NotFound) => {} 68 - // still failing after the retries: try the next cache 69 - Err(e) => { 70 - eprintln!("giving up on {c}/{h}.narinfo: {e}"); 71 - stats.add("narinfo_error", 0); 72 - } 73 - } 72 + Err(http::Error::NotFound) => continue, 73 + // still failing after the retries 74 + Err(e) => e.to_string(), 75 + }; 76 + // try the next cache 77 + eprintln!("giving up on {url}: {err}"); 78 + stats.add("narinfo_error", 0); 74 79 } 75 80 None 76 81 } ··· 86 91 let l = l?; 87 92 let l = l.trim(); 88 93 if !l.is_empty() { 89 - roots.insert(l.strip_prefix(STORE).unwrap_or(l).to_string()); 94 + let p = l.strip_prefix(STORE).unwrap_or(l); 95 + roots.insert(p.parse::<StorePath>().map_err(io::Error::other)?); 90 96 } 91 97 } 92 98 let caches = caches(); 93 99 let http = Http::new(caches.iter().map(String::as_str)).await; 94 100 let stats = Stats::new(None); 95 - let mut infos: BTreeMap<String, HashMap<String, String>> = BTreeMap::new(); 96 - let mut texts: HashMap<String, Vec<u8>> = HashMap::new(); 97 - let mut missing: BTreeSet<String> = BTreeSet::new(); 98 - let mut todo: Vec<String> = roots.iter().cloned().collect(); 101 + let mut infos: BTreeMap<StorePath, Found> = BTreeMap::new(); 102 + let mut missing: BTreeSet<StorePath> = BTreeSet::new(); 103 + let mut todo: Vec<StorePath> = roots.iter().cloned().collect(); 99 104 while !todo.is_empty() { 100 105 let (http, caches, stats) = (&http, &caches, &stats); 101 106 let mut res = stream::iter(std::mem::take(&mut todo)) 102 - .map(|n| async move { 103 - let i = narinfo(http, caches, &n, stats).await; 104 - (n, i) 107 + .map(|p| async move { 108 + let i = narinfo(http, caches, &p, stats).await; 109 + (p, i) 105 110 }) 106 111 .buffer_unordered(32); 107 112 let mut next = BTreeSet::new(); 108 - while let Some((name, i)) = res.next().await { 109 - let Some((i, text)) = i else { 110 - missing.insert(name.clone()); 113 + while let Some((path, i)) = res.next().await { 114 + let Some(i) = i else { 115 + missing.insert(path); 111 116 continue; 112 117 }; 113 - next.extend(references(&i).map(str::to_string)); 114 - infos.insert(name.clone(), i); 115 - texts.insert(name.clone(), text); 118 + next.extend(i.references().iter().cloned()); 119 + infos.insert(path, i); 116 120 } 117 121 todo = next 118 122 .into_iter() ··· 120 124 .collect(); 121 125 } 122 126 123 - let mut referrers: HashMap<String, Vec<String>> = HashMap::new(); 127 + let mut referrers: HashMap<&StorePath, Vec<&StorePath>> = HashMap::new(); 124 128 for (n, i) in &infos { 125 - for r in references(i).filter(|r| *r != n.as_str()) { 126 - referrers.entry(r.to_string()).or_default().push(n.clone()); 129 + for r in i.references().iter().filter(|r| *r != n) { 130 + referrers.entry(r).or_default().push(n); 127 131 } 128 132 } 129 133 let mut incomplete = BTreeSet::new(); 130 - let mut todo: Vec<&String> = missing.iter().collect(); 134 + let mut todo: Vec<&StorePath> = missing.iter().collect(); 131 135 while let Some(m) = todo.pop() { 132 136 for r in referrers.get(m).into_iter().flatten() { 133 - if incomplete.insert(r.clone()) { 137 + if incomplete.insert((*r).clone()) { 134 138 todo.push(r); 135 139 } 136 140 } 137 141 } 138 142 infos.retain(|n, _| !incomplete.contains(n)); 139 143 140 - let mut paths = BTreeMap::new(); 141 - for (n, i) in &infos { 142 - let get = |k: &str| { 143 - i.get(k) 144 - .cloned() 145 - .ok_or_else(|| io::Error::other(format!("{n}: narinfo without {k}"))) 146 - }; 147 - paths.insert( 148 - n.clone(), 149 - PathInfo { 150 - cache: get("cache")?, 151 - url: get("URL")?, 152 - compression: i 153 - .get("Compression") 154 - .cloned() 155 - .unwrap_or_else(|| "none".into()), 156 - nar_hash: get("NarHash")?, 157 - nar_size: get("NarSize")?.parse().map_err(io::Error::other)?, 158 - file_size: i.get("FileSize").and_then(|s| s.parse().ok()).unwrap_or(0), 159 - }, 160 - ); 161 - } 144 + let nar_hash = |i: &Found| { 145 + Hash::from(i.info.info.info.nar_hash) 146 + .as_base32() 147 + .to_string() 148 + }; 149 + let paths: BTreeMap<&StorePath, PathInfo> = infos 150 + .iter() 151 + .map(|(n, i)| { 152 + let ni = &i.info.info; 153 + let info = PathInfo { 154 + cache: i.cache.clone(), 155 + url: ni.url.clone().unwrap_or_default(), 156 + compression: ni.compression.clone().unwrap_or_else(|| "none".into()), 157 + nar_hash: nar_hash(i), 158 + nar_size: ni.info.nar_size, 159 + file_size: ni.download_size.unwrap_or(0), 160 + }; 161 + (n, info) 162 + }) 163 + .collect(); 162 164 std::fs::write(outdir.join("paths.json"), serde_json::to_vec(&paths)?)?; 163 165 164 166 let mut db = io::BufWriter::new(std::fs::File::create(outdir.join("db.dump"))?); 165 167 for (n, i) in &infos { 166 - let refs: Vec<&str> = references(i).collect(); 167 - let d = i.get("Deriver").map(String::as_str).unwrap_or(""); 168 - let deriver = if d.ends_with(".drv") { 169 - format!("{STORE}{d}") 170 - } else { 171 - String::new() 168 + let pi = &i.info.info.info; 169 + let deriver = match &pi.deriver { 170 + Some(d) if d.is_derivation() => format!("{STORE}{d}"), 171 + _ => String::new(), 172 172 }; 173 + let refs = i.references(); 173 174 writeln!( 174 175 db, 175 176 "{STORE}{n}\n{}\n{}\n{deriver}\n{}", 176 - i["NarHash"], 177 - i["NarSize"], 177 + nar_hash(i), 178 + pi.nar_size, 178 179 refs.len() 179 180 )?; 180 181 for r in refs { ··· 188 189 let dir = outdir.join("narinfo"); 189 190 std::fs::create_dir_all(&dir)?; 190 191 std::fs::write(dir.join("nix-cache-info"), "StoreDir: /nix/store\n")?; 191 - for n in infos.keys() { 192 - let h = n.split('-').next().unwrap_or(n); 193 - std::fs::write(dir.join(format!("{h}.narinfo")), &texts[n])?; 192 + for (n, i) in &infos { 193 + std::fs::write(dir.join(format!("{}.narinfo", n.hash())), &i.text)?; 194 194 } 195 195 196 - let mut by_cache: BTreeMap<String, u64> = BTreeMap::new(); 196 + let mut by_cache: BTreeMap<&str, u64> = BTreeMap::new(); 197 197 for p in paths.values() { 198 - *by_cache.entry(p.cache.clone()).or_default() += 1; 198 + *by_cache.entry(&p.cache).or_default() += 1; 199 199 } 200 200 let mut summary = serde_json::json!({ 201 201 "roots": roots.len(),