This repository has no description
7.4 kB
214 lines
1//! `closure OUT`: the closure of the store paths on stdin, from the caches'
2//! narinfos only.
3
4use std::collections::{BTreeMap, BTreeSet, HashMap};
5use std::io::{self, BufRead, Write};
6use std::path::Path;
7
8use futures_util::{stream, StreamExt};
9use harmonia_store_nar_info::{parse_narinfo_txt, NarInfo};
10use harmonia_store_path::{StoreDir, StorePath};
11use harmonia_utils_hash::{Hash, HashFormat};
12use serde::{Deserialize, Serialize};
13
14use crate::http::{self, Http};
15use crate::stats::Stats;
16use crate::STORE;
17
18/// Per store path, what the lazy layer needs to fetch it (OUT/paths.json).
19#[derive(Serialize, Deserialize, Clone, Debug)]
20#[serde(rename_all = "camelCase")]
21pub struct PathInfo {
22 pub cache: String,
23 pub url: String,
24 pub compression: String,
25 pub nar_hash: String,
26 pub nar_size: u64,
27 pub file_size: u64,
28}
29
30pub fn caches() -> Vec<String> {
31 std::env::var("LAZY_STORE_CACHES")
32 .unwrap_or_else(|_| "https://cache.nixos.org https://cache.lassul.us".into())
33 .split_whitespace()
34 .map(str::to_string)
35 .collect()
36}
37
38/// A narinfo, the cache it is from and its text.
39struct Found {
40 cache: String,
41 info: NarInfo,
42 text: Vec<u8>,
43}
44
45impl Found {
46 fn references(&self) -> &BTreeSet<StorePath> {
47 &self.info.info.info.references
48 }
49}
50
51/// The narinfo of `path` from the first cache that has it.
52async fn narinfo(http: &Http, caches: &[String], path: &StorePath, stats: &Stats) -> Option<Found> {
53 let h = path.hash();
54 for c in caches {
55 let url = format!("{c}/{h}.narinfo");
56 let err = match http.get(&url).await {
57 Ok((text, len)) => {
58 stats.add("narinfo", len as u64);
59 let parsed = std::str::from_utf8(&text)
60 .map_err(|e| e.to_string())
61 .and_then(|s| {
62 parse_narinfo_txt(&StoreDir::default(), s).map_err(|e| e.to_string())
63 });
64 match parsed {
65 Ok(info) => {
66 let cache = c.clone();
67 return Some(Found { cache, info, text });
68 }
69 Err(e) => e,
70 }
71 }
72 Err(http::Error::NotFound) => continue,
73 // still failing after the retries
74 Err(e) => e.to_string(),
75 };
76 // try the next cache
77 eprintln!("giving up on {url}: {err}");
78 stats.add("narinfo_error", 0);
79 }
80 None
81}
82
83/// Writes OUT/paths.json (per path: cache, NAR url, compression, hash),
84/// OUT/db.dump (input for `nix-store --load-db`), OUT/narinfo (a binary
85/// cache of just the narinfos) and OUT/closure.json.
86/// Paths that are in none of the caches, and everything referring to them,
87/// are left out: a path is registered only with its whole closure.
88pub async fn closure(outdir: &Path) -> io::Result<()> {
89 let mut roots = BTreeSet::new();
90 for l in io::stdin().lock().lines() {
91 let l = l?;
92 let l = l.trim();
93 if !l.is_empty() {
94 let p = l.strip_prefix(STORE).unwrap_or(l);
95 roots.insert(p.parse::<StorePath>().map_err(io::Error::other)?);
96 }
97 }
98 let caches = caches();
99 let http = Http::new(caches.iter().map(String::as_str)).await;
100 let stats = Stats::new(None);
101 let mut infos: BTreeMap<StorePath, Found> = BTreeMap::new();
102 let mut missing: BTreeSet<StorePath> = BTreeSet::new();
103 let mut todo: Vec<StorePath> = roots.iter().cloned().collect();
104 while !todo.is_empty() {
105 let (http, caches, stats) = (&http, &caches, &stats);
106 let mut res = stream::iter(std::mem::take(&mut todo))
107 .map(|p| async move {
108 let i = narinfo(http, caches, &p, stats).await;
109 (p, i)
110 })
111 .buffer_unordered(32);
112 let mut next = BTreeSet::new();
113 while let Some((path, i)) = res.next().await {
114 let Some(i) = i else {
115 missing.insert(path);
116 continue;
117 };
118 next.extend(i.references().iter().cloned());
119 infos.insert(path, i);
120 }
121 todo = next
122 .into_iter()
123 .filter(|r| !infos.contains_key(r) && !missing.contains(r))
124 .collect();
125 }
126
127 let mut referrers: HashMap<&StorePath, Vec<&StorePath>> = HashMap::new();
128 for (n, i) in &infos {
129 for r in i.references().iter().filter(|r| *r != n) {
130 referrers.entry(r).or_default().push(n);
131 }
132 }
133 let mut incomplete = BTreeSet::new();
134 let mut todo: Vec<&StorePath> = missing.iter().collect();
135 while let Some(m) = todo.pop() {
136 for r in referrers.get(m).into_iter().flatten() {
137 if incomplete.insert((*r).clone()) {
138 todo.push(r);
139 }
140 }
141 }
142 infos.retain(|n, _| !incomplete.contains(n));
143
144 let nar_hash = |i: &Found| {
145 Hash::from(i.info.info.info.nar_hash)
146 .as_base32()
147 .to_string()
148 };
149 let paths: BTreeMap<&StorePath, PathInfo> = infos
150 .iter()
151 .map(|(n, i)| {
152 let ni = &i.info.info;
153 let info = PathInfo {
154 cache: i.cache.clone(),
155 url: ni.url.clone().unwrap_or_default(),
156 compression: ni.compression.clone().unwrap_or_else(|| "none".into()),
157 nar_hash: nar_hash(i),
158 nar_size: ni.info.nar_size,
159 file_size: ni.download_size.unwrap_or(0),
160 };
161 (n, info)
162 })
163 .collect();
164 std::fs::write(outdir.join("paths.json"), serde_json::to_vec(&paths)?)?;
165
166 let mut db = io::BufWriter::new(std::fs::File::create(outdir.join("db.dump"))?);
167 for (n, i) in &infos {
168 let pi = &i.info.info.info;
169 let deriver = match &pi.deriver {
170 Some(d) if d.is_derivation() => format!("{STORE}{d}"),
171 _ => String::new(),
172 };
173 let refs = i.references();
174 writeln!(
175 db,
176 "{STORE}{n}\n{}\n{}\n{deriver}\n{}",
177 nar_hash(i),
178 pi.nar_size,
179 refs.len()
180 )?;
181 for r in refs {
182 writeln!(db, "{STORE}{r}")?;
183 }
184 }
185 db.flush()?;
186
187 // the narinfos as a binary cache, for their signatures: `nix-store
188 // --load-db` registers none, `nix store copy-sigs` adds them
189 let dir = outdir.join("narinfo");
190 std::fs::create_dir_all(&dir)?;
191 std::fs::write(dir.join("nix-cache-info"), "StoreDir: /nix/store\n")?;
192 for (n, i) in &infos {
193 std::fs::write(dir.join(format!("{}.narinfo", n.hash())), &i.text)?;
194 }
195
196 let mut by_cache: BTreeMap<&str, u64> = BTreeMap::new();
197 for p in paths.values() {
198 *by_cache.entry(&p.cache).or_default() += 1;
199 }
200 let mut summary = serde_json::json!({
201 "roots": roots.len(),
202 "paths": infos.len(),
203 "missing": missing,
204 "incomplete": incomplete.len(),
205 "by_cache": by_cache,
206 "nar_size": paths.values().map(|p| p.nar_size).sum::<u64>(),
207 "file_size": paths.values().map(|p| p.file_size).sum::<u64>(),
208 });
209 summary.as_object_mut().unwrap().extend(stats.counters());
210 let s = serde_json::to_string_pretty(&summary)?;
211 std::fs::write(outdir.join("closure.json"), &s)?;
212 println!("{s}");
213 Ok(())
214}