This repository has no description
11 kB
366 lines
1//! NAR unpacking and store path listings.
2
3use std::collections::BTreeMap;
4use std::fs;
5use std::io::{self, Read, Write};
6use std::os::unix::ffi::OsStrExt;
7use std::os::unix::fs::{symlink, MetadataExt, PermissionsExt};
8use std::path::Path;
9
10use harmonia_file_core::{FileSystemObject, FileTree};
11use harmonia_utils_base_encoding::base32;
12use serde::Deserialize;
13use sha2::{Digest, Sha256};
14
15/// Does `digest` match a narinfo NarHash (`sha256:<nix32>` or base16)?
16pub fn hash_matches(expected: &str, digest: &[u8]) -> bool {
17 let Some(h) = expected.strip_prefix("sha256:") else {
18 return false;
19 };
20 let hex: String = digest.iter().map(|b| format!("{b:02x}")).collect();
21 h == base32::encode_string(digest) || h == hex
22}
23
24/// The metadata of a store path, as in the caches' `.ls` listings.
25#[derive(Debug, Clone, PartialEq)]
26pub enum Tree {
27 Dir(BTreeMap<String, Tree>),
28 File { size: u64, exec: bool },
29 Symlink(String),
30}
31
32impl Tree {
33 /// Parse a `.ls` listing (`{"version": 1, "root": {...}}`).
34 pub fn from_listing(body: &[u8]) -> io::Result<Tree> {
35 #[derive(Deserialize)]
36 struct Size {
37 size: u64,
38 }
39 #[derive(Deserialize)]
40 struct Listing {
41 root: FileTree<Size>,
42 }
43 fn tree(FileTree(t): FileTree<Size>) -> Tree {
44 match t {
45 FileSystemObject::Directory(d) => Tree::Dir(
46 d.entries
47 .into_iter()
48 .map(|(k, c)| (k, tree(*c)))
49 .collect(),
50 ),
51 FileSystemObject::Regular(r) => Tree::File {
52 size: r.contents.size,
53 exec: r.executable,
54 },
55 FileSystemObject::Symlink(s) => Tree::Symlink(s.target),
56 }
57 }
58 let l: Listing = serde_json::from_slice(body).map_err(io::Error::other)?;
59 Ok(tree(l.root))
60 }
61
62 /// Listing of an unpacked store path (for paths without `.ls`).
63 pub fn from_path(p: &Path) -> io::Result<Tree> {
64 let m = fs::symlink_metadata(p)?;
65 let t = m.file_type();
66 Ok(if t.is_dir() {
67 let mut e = BTreeMap::new();
68 for d in fs::read_dir(p)? {
69 let d = d?;
70 let name = d.file_name().into_string().map_err(|n| {
71 io::Error::other(format!("non-UTF-8 name {n:?} in {}", p.display()))
72 })?;
73 e.insert(name, Tree::from_path(&d.path())?);
74 }
75 Tree::Dir(e)
76 } else if t.is_symlink() {
77 Tree::Symlink(fs::read_link(p)?.to_string_lossy().into_owned())
78 } else {
79 Tree::File {
80 size: m.size(),
81 exec: m.mode() & 0o100 != 0,
82 }
83 })
84 }
85}
86
87/// Reads through to `inner`, hashing and counting what passes.
88pub struct HashingReader<R> {
89 inner: R,
90 pub hasher: Sha256,
91 pub n: u64,
92}
93
94impl<R: Read> Read for HashingReader<R> {
95 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
96 let n = self.inner.read(buf)?;
97 self.hasher.update(&buf[..n]);
98 self.n += n as u64;
99 Ok(n)
100 }
101}
102
103/// Counts the bytes read from `inner` (the compressed download size).
104pub struct CountingReader<R> {
105 pub inner: R,
106 pub n: u64,
107}
108
109impl<R: Read> Read for CountingReader<R> {
110 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
111 let n = self.inner.read(buf)?;
112 self.n += n as u64;
113 Ok(n)
114 }
115}
116
117fn bad(msg: impl Into<String>) -> io::Error {
118 io::Error::new(io::ErrorKind::InvalidData, format!("NAR: {}", msg.into()))
119}
120
121struct Nar<R> {
122 r: HashingReader<R>,
123}
124
125impl<R: Read> Nar<R> {
126 fn int(&mut self) -> io::Result<u64> {
127 let mut b = [0u8; 8];
128 self.r.read_exact(&mut b)?;
129 Ok(u64::from_le_bytes(b))
130 }
131
132 fn pad(&mut self, n: u64) -> io::Result<()> {
133 let p = ((8 - n % 8) % 8) as usize;
134 let mut b = [0u8; 8];
135 self.r.read_exact(&mut b[..p])?;
136 if b[..p].iter().any(|&x| x != 0) {
137 return Err(bad("non-zero padding"));
138 }
139 Ok(())
140 }
141
142 fn str(&mut self, limit: u64) -> io::Result<Vec<u8>> {
143 let n = self.int()?;
144 if n > limit {
145 return Err(bad("string too long"));
146 }
147 let mut s = vec![0u8; n as usize];
148 self.r.read_exact(&mut s)?;
149 self.pad(n)?;
150 Ok(s)
151 }
152
153 fn expect(&mut self, s: &[u8]) -> io::Result<()> {
154 let got = self.str(4096)?;
155 if got != s {
156 return Err(bad(format!(
157 "expected {:?}, got {:?}",
158 String::from_utf8_lossy(s),
159 String::from_utf8_lossy(&got)
160 )));
161 }
162 Ok(())
163 }
164
165 fn node(&mut self, dest: &Path) -> io::Result<()> {
166 self.expect(b"(")?;
167 self.expect(b"type")?;
168 match &self.str(4096)?[..] {
169 b"regular" => {
170 let mut tag = self.str(4096)?;
171 let mut mode = 0o444;
172 if tag == b"executable" {
173 self.expect(b"")?;
174 tag = self.str(4096)?;
175 mode = 0o555;
176 }
177 if tag != b"contents" {
178 return Err(bad("expected contents"));
179 }
180 let n = self.int()?;
181 let mut out = io::BufWriter::with_capacity(1 << 20, fs::File::create(dest)?);
182 let copied = io::copy(&mut (&mut self.r).take(n), &mut out)?;
183 if copied != n {
184 return Err(bad("short file contents"));
185 }
186 out.flush()?;
187 self.pad(n)?;
188 fs::set_permissions(dest, fs::Permissions::from_mode(mode))?;
189 self.expect(b")")?;
190 }
191 b"symlink" => {
192 self.expect(b"target")?;
193 let t = self.str(4096)?;
194 symlink(std::ffi::OsStr::from_bytes(&t), dest)?;
195 self.expect(b")")?;
196 }
197 b"directory" => {
198 fs::create_dir(dest)?;
199 loop {
200 let tag = self.str(4096)?;
201 if tag == b")" {
202 break;
203 }
204 if tag != b"entry" {
205 return Err(bad("expected entry"));
206 }
207 self.expect(b"(")?;
208 self.expect(b"name")?;
209 let name = self.str(255)?;
210 if name.is_empty() || name == b"." || name == b".." || name.contains(&b'/') {
211 return Err(bad("bad entry name"));
212 }
213 self.expect(b"node")?;
214 let name = String::from_utf8(name).map_err(|_| bad("non-UTF-8 name"))?;
215 self.node(&dest.join(name))?;
216 self.expect(b")")?;
217 }
218 fs::set_permissions(dest, fs::Permissions::from_mode(0o555))?;
219 }
220 t => {
221 return Err(bad(format!(
222 "unknown type {:?}",
223 String::from_utf8_lossy(t)
224 )))
225 }
226 }
227 Ok(())
228 }
229}
230
231/// Unpack the NAR read from `r` to `dest` (which must not exist). Returns
232/// the NAR's sha256 and size; checking them is up to the caller. Fails if
233/// anything follows the NAR.
234pub fn unpack(r: impl Read, dest: &Path) -> io::Result<(Vec<u8>, u64)> {
235 let mut nar = Nar {
236 r: HashingReader {
237 inner: r,
238 hasher: Sha256::new(),
239 n: 0,
240 },
241 };
242 nar.expect(b"nix-archive-1")?;
243 nar.node(dest)?;
244 let mut rest = [0u8; 1];
245 if nar.r.inner.read(&mut rest)? != 0 {
246 return Err(bad("trailing data"));
247 }
248 let n = nar.r.n;
249 Ok((nar.r.hasher.finalize().to_vec(), n))
250}
251
252/// Remove a (partially) unpacked tree, including read-only directories.
253pub fn remove(p: &Path) -> io::Result<()> {
254 let m = match fs::symlink_metadata(p) {
255 Ok(m) => m,
256 Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
257 Err(e) => return Err(e),
258 };
259 if m.is_dir() {
260 fs::set_permissions(p, fs::Permissions::from_mode(0o755))?;
261 for d in fs::read_dir(p)? {
262 remove(&d?.path())?;
263 }
264 fs::remove_dir(p)
265 } else {
266 fs::remove_file(p)
267 }
268}
269
270#[cfg(test)]
271mod tests {
272 use super::*;
273
274 fn s(out: &mut Vec<u8>, b: &[u8]) {
275 out.extend_from_slice(&(b.len() as u64).to_le_bytes());
276 out.extend_from_slice(b);
277 out.resize(out.len() + (8 - b.len() % 8) % 8, 0);
278 }
279
280 fn nar() -> Vec<u8> {
281 let mut o = Vec::new();
282 for t in [&b"nix-archive-1"[..], b"(", b"type", b"directory"] {
283 s(&mut o, t);
284 }
285 for t in [&b"entry"[..], b"(", b"name", b"bin", b"node", b"("] {
286 s(&mut o, t);
287 }
288 for t in [
289 &b"type"[..],
290 b"regular",
291 b"executable",
292 b"",
293 b"contents",
294 b"#!/bin/sh\n",
295 ] {
296 s(&mut o, t);
297 }
298 for t in [&b")"[..], b")"] {
299 s(&mut o, t);
300 }
301 for t in [&b"entry"[..], b"(", b"name", b"link", b"node", b"("] {
302 s(&mut o, t);
303 }
304 for t in [
305 &b"type"[..],
306 b"symlink",
307 b"target",
308 b"bin",
309 b")",
310 b")",
311 b")",
312 ] {
313 s(&mut o, t);
314 }
315 o
316 }
317
318 #[test]
319 fn hash_matches_nix() {
320 // nix hash convert --to nix32 sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=
321 let d = Sha256::digest(b"");
322 assert!(hash_matches(
323 "sha256:0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73",
324 &d
325 ));
326 assert!(hash_matches(
327 "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
328 &d
329 ));
330 assert!(!hash_matches(
331 "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73",
332 &d
333 ));
334 }
335
336 #[test]
337 fn unpack_and_list() {
338 let dir = std::env::temp_dir().join(format!("lazy-store-test-{}", std::process::id()));
339 let _ = remove(&dir);
340 let data = nar();
341 let (h, n) = unpack(&data[..], &dir).unwrap();
342 assert_eq!(n, data.len() as u64);
343 assert_eq!(h, Sha256::digest(&data).to_vec());
344 let t = Tree::from_path(&dir).unwrap();
345 let mut e = BTreeMap::new();
346 e.insert(
347 "bin".into(),
348 Tree::File {
349 size: 10,
350 exec: true,
351 },
352 );
353 e.insert("link".into(), Tree::Symlink("bin".into()));
354 assert_eq!(t, Tree::Dir(e));
355 let ls = br#"{"version":1,"root":{"type":"directory","entries":{
356 "bin":{"type":"regular","size":10,"executable":true,"narOffset":96},
357 "link":{"type":"symlink","target":"bin"}}}}"#;
358 assert_eq!(Tree::from_listing(ls).unwrap(), t);
359 remove(&dir).unwrap();
360
361 let mut trailing = data.clone();
362 trailing.push(0);
363 assert!(unpack(&trailing[..], &dir).is_err());
364 remove(&dir).unwrap();
365 }
366}