This repository has no description
7.4 kB
214 lines
1//! `closure OUT`: the closure of the store paths on stdin, from the caches'
2//! narinfos only.
3
4use std::collections::{BTreeMap, BTreeSet, HashMap};
5use std::io::{self, BufRead, Write};
6use std::path::Path;
7
8use futures_util::{stream, StreamExt};
9use serde::{Deserialize, Serialize};
10
11use crate::http::{self, Http};
12use crate::stats::Stats;
13use crate::STORE;
14
15/// Per store path, what the lazy layer needs to fetch it (OUT/paths.json).
16#[derive(Serialize, Deserialize, Clone, Debug)]
17#[serde(rename_all = "camelCase")]
18pub struct PathInfo {
19 pub cache: String,
20 pub url: String,
21 pub compression: String,
22 pub nar_hash: String,
23 pub nar_size: u64,
24 pub file_size: u64,
25}
26
27pub fn caches() -> Vec<String> {
28 std::env::var("LAZY_STORE_CACHES")
29 .unwrap_or_else(|_| "https://cache.nixos.org https://cache.lassul.us".into())
30 .split_whitespace()
31 .map(str::to_string)
32 .collect()
33}
34
35fn parse_narinfo(text: &[u8]) -> HashMap<String, String> {
36 String::from_utf8_lossy(text)
37 .lines()
38 .filter_map(|l| l.split_once(": "))
39 .map(|(k, v)| (k.to_string(), v.to_string()))
40 .collect()
41}
42
43fn references(i: &HashMap<String, String>) -> impl Iterator<Item = &str> {
44 i.get("References")
45 .map(String::as_str)
46 .unwrap_or("")
47 .split_whitespace()
48}
49
50/// The parsed narinfo of `name` from the first cache that has it, and its
51/// text.
52async fn narinfo(
53 http: &Http,
54 caches: &[String],
55 name: &str,
56 stats: &Stats,
57) -> Option<(HashMap<String, String>, Vec<u8>)> {
58 let h = name.split('-').next().unwrap_or(name);
59 for c in caches {
60 match http.get(&format!("{c}/{h}.narinfo")).await {
61 Ok((body, len)) => {
62 stats.add("narinfo", len as u64);
63 let mut i = parse_narinfo(&body);
64 i.insert("cache".into(), c.clone());
65 return Some((i, body));
66 }
67 Err(http::Error::NotFound) => {}
68 // still failing after the retries: try the next cache
69 Err(e) => {
70 eprintln!("giving up on {c}/{h}.narinfo: {e}");
71 stats.add("narinfo_error", 0);
72 }
73 }
74 }
75 None
76}
77
78/// Writes OUT/paths.json (per path: cache, NAR url, compression, hash),
79/// OUT/db.dump (input for `nix-store --load-db`), OUT/narinfo (a binary
80/// cache of just the narinfos) and OUT/closure.json.
81/// Paths that are in none of the caches, and everything referring to them,
82/// are left out: a path is registered only with its whole closure.
83pub async fn closure(outdir: &Path) -> io::Result<()> {
84 let mut roots = BTreeSet::new();
85 for l in io::stdin().lock().lines() {
86 let l = l?;
87 let l = l.trim();
88 if !l.is_empty() {
89 roots.insert(l.strip_prefix(STORE).unwrap_or(l).to_string());
90 }
91 }
92 let caches = caches();
93 let http = Http::new(caches.iter().map(String::as_str)).await;
94 let stats = Stats::new(None);
95 let mut infos: BTreeMap<String, HashMap<String, String>> = BTreeMap::new();
96 let mut texts: HashMap<String, Vec<u8>> = HashMap::new();
97 let mut missing: BTreeSet<String> = BTreeSet::new();
98 let mut todo: Vec<String> = roots.iter().cloned().collect();
99 while !todo.is_empty() {
100 let (http, caches, stats) = (&http, &caches, &stats);
101 let mut res = stream::iter(std::mem::take(&mut todo))
102 .map(|n| async move {
103 let i = narinfo(http, caches, &n, stats).await;
104 (n, i)
105 })
106 .buffer_unordered(32);
107 let mut next = BTreeSet::new();
108 while let Some((name, i)) = res.next().await {
109 let Some((i, text)) = i else {
110 missing.insert(name.clone());
111 continue;
112 };
113 next.extend(references(&i).map(str::to_string));
114 infos.insert(name.clone(), i);
115 texts.insert(name.clone(), text);
116 }
117 todo = next
118 .into_iter()
119 .filter(|r| !infos.contains_key(r) && !missing.contains(r))
120 .collect();
121 }
122
123 let mut referrers: HashMap<String, Vec<String>> = HashMap::new();
124 for (n, i) in &infos {
125 for r in references(i).filter(|r| *r != n.as_str()) {
126 referrers.entry(r.to_string()).or_default().push(n.clone());
127 }
128 }
129 let mut incomplete = BTreeSet::new();
130 let mut todo: Vec<&String> = missing.iter().collect();
131 while let Some(m) = todo.pop() {
132 for r in referrers.get(m).into_iter().flatten() {
133 if incomplete.insert(r.clone()) {
134 todo.push(r);
135 }
136 }
137 }
138 infos.retain(|n, _| !incomplete.contains(n));
139
140 let mut paths = BTreeMap::new();
141 for (n, i) in &infos {
142 let get = |k: &str| {
143 i.get(k)
144 .cloned()
145 .ok_or_else(|| io::Error::other(format!("{n}: narinfo without {k}")))
146 };
147 paths.insert(
148 n.clone(),
149 PathInfo {
150 cache: get("cache")?,
151 url: get("URL")?,
152 compression: i
153 .get("Compression")
154 .cloned()
155 .unwrap_or_else(|| "none".into()),
156 nar_hash: get("NarHash")?,
157 nar_size: get("NarSize")?.parse().map_err(io::Error::other)?,
158 file_size: i.get("FileSize").and_then(|s| s.parse().ok()).unwrap_or(0),
159 },
160 );
161 }
162 std::fs::write(outdir.join("paths.json"), serde_json::to_vec(&paths)?)?;
163
164 let mut db = io::BufWriter::new(std::fs::File::create(outdir.join("db.dump"))?);
165 for (n, i) in &infos {
166 let refs: Vec<&str> = references(i).collect();
167 let d = i.get("Deriver").map(String::as_str).unwrap_or("");
168 let deriver = if d.ends_with(".drv") {
169 format!("{STORE}{d}")
170 } else {
171 String::new()
172 };
173 writeln!(
174 db,
175 "{STORE}{n}\n{}\n{}\n{deriver}\n{}",
176 i["NarHash"],
177 i["NarSize"],
178 refs.len()
179 )?;
180 for r in refs {
181 writeln!(db, "{STORE}{r}")?;
182 }
183 }
184 db.flush()?;
185
186 // the narinfos as a binary cache, for their signatures: `nix-store
187 // --load-db` registers none, `nix store copy-sigs` adds them
188 let dir = outdir.join("narinfo");
189 std::fs::create_dir_all(&dir)?;
190 std::fs::write(dir.join("nix-cache-info"), "StoreDir: /nix/store\n")?;
191 for n in infos.keys() {
192 let h = n.split('-').next().unwrap_or(n);
193 std::fs::write(dir.join(format!("{h}.narinfo")), &texts[n])?;
194 }
195
196 let mut by_cache: BTreeMap<String, u64> = BTreeMap::new();
197 for p in paths.values() {
198 *by_cache.entry(p.cache.clone()).or_default() += 1;
199 }
200 let mut summary = serde_json::json!({
201 "roots": roots.len(),
202 "paths": infos.len(),
203 "missing": missing,
204 "incomplete": incomplete.len(),
205 "by_cache": by_cache,
206 "nar_size": paths.values().map(|p| p.nar_size).sum::<u64>(),
207 "file_size": paths.values().map(|p| p.file_size).sum::<u64>(),
208 });
209 summary.as_object_mut().unwrap().extend(stats.counters());
210 let s = serde_json::to_string_pretty(&summary)?;
211 std::fs::write(outdir.join("closure.json"), &s)?;
212 println!("{s}");
213 Ok(())
214}