This repository has no description
0

Configure Feed

Select the types of activity you want to include in your feed.

tools.lazy-store: unpack NARs with harmonia-file-nar

Our NAR reader goes: NARs are parsed by harmonia-file-nar's parse_nar
and written by its restore (async, bounded queue to one writer thread;
it also checks entry names, sorting and padding like Nix). Files come
out with the umask's permissions, so a pass afterwards sets the store's
0444/0555 like our reader did.

The NAR hash is computed with harmonia-utils-hash (aws-lc sha256) in a
tokio-util InspectReader instead of our HashingReader and sha2, and
narinfo NarHashes are parsed with harmonia (any encoding with the
sha256: prefix) instead of comparing base32/base16 strings. Data after
the NAR counts into the size, so the NarSize check still rejects it.

The .ls listings are harmonia-file-core FileTrees now instead of our
Tree enum converted from them. The test unpacks a NAR made by
harmonia's NarByteStream instead of a hand-written one.

+361 -363
+213 -46
tools/lazy-store/Cargo.lock
··· 33 33 ] 34 34 35 35 [[package]] 36 + name = "arrayvec" 37 + version = "0.7.8" 38 + source = "registry+https://github.com/rust-lang/crates.io-index" 39 + checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" 40 + 41 + [[package]] 36 42 name = "async-compression" 37 43 version = "0.4.51" 38 44 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 63 69 checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" 64 70 dependencies = [ 65 71 "aws-lc-sys", 72 + "untrusted 0.7.1", 66 73 "zeroize", 67 74 ] 68 75 ··· 98 105 checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" 99 106 100 107 [[package]] 101 - name = "block-buffer" 102 - version = "0.10.4" 108 + name = "blake3" 109 + version = "1.8.7" 103 110 source = "registry+https://github.com/rust-lang/crates.io-index" 104 - checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" 111 + checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" 105 112 dependencies = [ 106 - "generic-array", 113 + "arrayvec", 114 + "cc", 115 + "cfg-if", 116 + "constant_time_eq", 117 + "cpufeatures", 118 + ] 119 + 120 + [[package]] 121 + name = "block-buffer" 122 + version = "0.12.1" 123 + source = "registry+https://github.com/rust-lang/crates.io-index" 124 + checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" 125 + dependencies = [ 126 + "hybrid-array", 107 127 ] 108 128 109 129 [[package]] ··· 125 145 dependencies = [ 126 146 "alloc-no-stdlib", 127 147 "alloc-stdlib", 148 + ] 149 + 150 + [[package]] 151 + name = "bstr" 152 + version = "1.13.1" 153 + source = "registry+https://github.com/rust-lang/crates.io-index" 154 + checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" 155 + dependencies = [ 156 + "memchr", 157 + "regex-automata", 158 + "serde_core", 128 159 ] 129 160 130 161 [[package]] ··· 188 219 checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" 189 220 dependencies = [ 190 221 "cfg-if", 191 - "cpufeatures 0.3.1", 222 + "cpufeatures", 192 223 "rand_core", 193 224 ] 194 225 ··· 234 265 checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" 235 266 236 267 [[package]] 268 + name = "const-oid" 269 + version = "0.10.2" 270 + source = "registry+https://github.com/rust-lang/crates.io-index" 271 + checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" 272 + 273 + [[package]] 274 + name = "constant_time_eq" 275 + version = "0.4.2" 276 + source = "registry+https://github.com/rust-lang/crates.io-index" 277 + checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" 278 + 279 + [[package]] 237 280 name = "convert_case" 238 281 version = "0.10.0" 239 282 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 260 303 261 304 [[package]] 262 305 name = "cpufeatures" 263 - version = "0.2.17" 264 - source = "registry+https://github.com/rust-lang/crates.io-index" 265 - checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" 266 - dependencies = [ 267 - "libc", 268 - ] 269 - 270 - [[package]] 271 - name = "cpufeatures" 272 306 version = "0.3.1" 273 307 source = "registry+https://github.com/rust-lang/crates.io-index" 274 308 checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" ··· 287 321 288 322 [[package]] 289 323 name = "crypto-common" 290 - version = "0.1.7" 324 + version = "0.2.2" 291 325 source = "registry+https://github.com/rust-lang/crates.io-index" 292 - checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" 326 + checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" 293 327 dependencies = [ 294 - "generic-array", 295 - "typenum", 328 + "hybrid-array", 296 329 ] 297 330 298 331 [[package]] ··· 326 359 327 360 [[package]] 328 361 name = "digest" 329 - version = "0.10.7" 362 + version = "0.11.3" 330 363 source = "registry+https://github.com/rust-lang/crates.io-index" 331 - checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" 364 + checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" 332 365 dependencies = [ 333 366 "block-buffer", 367 + "const-oid", 334 368 "crypto-common", 335 369 ] 336 370 ··· 362 396 version = "1.0.3" 363 397 source = "registry+https://github.com/rust-lang/crates.io-index" 364 398 checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" 399 + 400 + [[package]] 401 + name = "errno" 402 + version = "0.3.14" 403 + source = "registry+https://github.com/rust-lang/crates.io-index" 404 + checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" 405 + dependencies = [ 406 + "libc", 407 + "windows-sys 0.52.0", 408 + ] 365 409 366 410 [[package]] 367 411 name = "find-msvc-tools" ··· 422 466 "libc", 423 467 "log", 424 468 "mio 0.8.11", 425 - "nix", 469 + "nix 0.24.3", 426 470 "vm-memory", 427 471 "vmm-sys-util", 428 472 ] ··· 537 581 ] 538 582 539 583 [[package]] 540 - name = "generic-array" 541 - version = "0.14.7" 542 - source = "registry+https://github.com/rust-lang/crates.io-index" 543 - checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" 544 - dependencies = [ 545 - "typenum", 546 - "version_check", 547 - ] 548 - 549 - [[package]] 550 584 name = "getrandom" 551 585 version = "0.2.17" 552 586 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 603 637 ] 604 638 605 639 [[package]] 640 + name = "harmonia-file-nar" 641 + version = "3.3.0" 642 + source = "git+https://github.com/nix-community/harmonia?rev=3edb37ae36b561c2d395980170c70bbce292bbc5#3edb37ae36b561c2d395980170c70bbce292bbc5" 643 + dependencies = [ 644 + "bstr", 645 + "bytes", 646 + "derive_more", 647 + "futures-core", 648 + "futures-sink", 649 + "futures-util", 650 + "harmonia-file-core", 651 + "harmonia-utils-io", 652 + "nix 0.31.3", 653 + "pin-project-lite", 654 + "serde", 655 + "serde_json", 656 + "thiserror", 657 + "tokio", 658 + "tokio-util", 659 + "tracing", 660 + "walkdir", 661 + ] 662 + 663 + [[package]] 606 664 name = "harmonia-utils-base-encoding" 607 665 version = "0.0.0-alpha.0" 608 666 source = "git+https://github.com/nix-community/harmonia?rev=3edb37ae36b561c2d395980170c70bbce292bbc5#3edb37ae36b561c2d395980170c70bbce292bbc5" ··· 610 668 "data-encoding", 611 669 "derive_more", 612 670 "serde", 671 + ] 672 + 673 + [[package]] 674 + name = "harmonia-utils-hash" 675 + version = "0.0.0-alpha.0" 676 + source = "git+https://github.com/nix-community/harmonia?rev=3edb37ae36b561c2d395980170c70bbce292bbc5#3edb37ae36b561c2d395980170c70bbce292bbc5" 677 + dependencies = [ 678 + "aws-lc-rs", 679 + "blake3", 680 + "data-encoding", 681 + "derive_more", 682 + "harmonia-utils-base-encoding", 683 + "md5", 684 + "pin-project-lite", 685 + "serde", 686 + "sha1", 687 + "sha2", 688 + "thiserror", 689 + ] 690 + 691 + [[package]] 692 + name = "harmonia-utils-io" 693 + version = "0.0.0-alpha.0" 694 + source = "git+https://github.com/nix-community/harmonia?rev=3edb37ae36b561c2d395980170c70bbce292bbc5#3edb37ae36b561c2d395980170c70bbce292bbc5" 695 + dependencies = [ 696 + "bytes", 697 + "futures-util", 698 + "harmonia-utils-hash", 699 + "libc", 700 + "nix 0.31.3", 701 + "pin-project-lite", 702 + "tokio", 613 703 ] 614 704 615 705 [[package]] ··· 656 746 version = "1.10.1" 657 747 source = "registry+https://github.com/rust-lang/crates.io-index" 658 748 checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" 749 + 750 + [[package]] 751 + name = "hybrid-array" 752 + version = "0.4.15" 753 + source = "registry+https://github.com/rust-lang/crates.io-index" 754 + checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" 755 + dependencies = [ 756 + "typenum", 757 + ] 659 758 660 759 [[package]] 661 760 name = "hyper" ··· 921 1020 "fuse-backend-rs", 922 1021 "futures-util", 923 1022 "harmonia-file-core", 924 - "harmonia-utils-base-encoding", 1023 + "harmonia-file-nar", 1024 + "harmonia-utils-hash", 925 1025 "libc", 926 1026 "log", 927 1027 "reqwest", 928 1028 "serde", 929 1029 "serde_json", 930 - "sha2", 931 1030 "tokio", 932 1031 "tokio-util", 933 1032 "webpki-root-certs", ··· 984 1083 checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" 985 1084 986 1085 [[package]] 1086 + name = "md5" 1087 + version = "0.8.1" 1088 + source = "registry+https://github.com/rust-lang/crates.io-index" 1089 + checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c" 1090 + 1091 + [[package]] 987 1092 name = "memchr" 988 1093 version = "2.8.3" 989 1094 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 994 1099 version = "0.6.5" 995 1100 source = "registry+https://github.com/rust-lang/crates.io-index" 996 1101 checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" 1102 + dependencies = [ 1103 + "autocfg", 1104 + ] 1105 + 1106 + [[package]] 1107 + name = "memoffset" 1108 + version = "0.9.1" 1109 + source = "registry+https://github.com/rust-lang/crates.io-index" 1110 + checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" 997 1111 dependencies = [ 998 1112 "autocfg", 999 1113 ] ··· 1049 1163 "bitflags 1.3.2", 1050 1164 "cfg-if", 1051 1165 "libc", 1052 - "memoffset", 1166 + "memoffset 0.6.5", 1167 + ] 1168 + 1169 + [[package]] 1170 + name = "nix" 1171 + version = "0.31.3" 1172 + source = "registry+https://github.com/rust-lang/crates.io-index" 1173 + checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" 1174 + dependencies = [ 1175 + "bitflags 2.13.2", 1176 + "cfg-if", 1177 + "cfg_aliases", 1178 + "libc", 1179 + "memoffset 0.9.1", 1053 1180 ] 1054 1181 1055 1182 [[package]] ··· 1209 1336 ] 1210 1337 1211 1338 [[package]] 1339 + name = "regex-automata" 1340 + version = "0.4.18" 1341 + source = "registry+https://github.com/rust-lang/crates.io-index" 1342 + checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" 1343 + 1344 + [[package]] 1212 1345 name = "reqwest" 1213 1346 version = "0.13.5" 1214 1347 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1257 1390 "cfg-if", 1258 1391 "getrandom 0.2.17", 1259 1392 "libc", 1260 - "untrusted", 1393 + "untrusted 0.9.0", 1261 1394 "windows-sys 0.52.0", 1262 1395 ] 1263 1396 ··· 1348 1481 "aws-lc-rs", 1349 1482 "ring", 1350 1483 "rustls-pki-types", 1351 - "untrusted", 1484 + "untrusted 0.9.0", 1352 1485 ] 1353 1486 1354 1487 [[package]] ··· 1448 1581 ] 1449 1582 1450 1583 [[package]] 1451 - name = "sha2" 1452 - version = "0.10.9" 1584 + name = "sha1" 1585 + version = "0.11.0" 1453 1586 source = "registry+https://github.com/rust-lang/crates.io-index" 1454 - checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" 1587 + checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" 1455 1588 dependencies = [ 1456 1589 "cfg-if", 1457 - "cpufeatures 0.2.17", 1590 + "cpufeatures", 1591 + "digest", 1592 + ] 1593 + 1594 + [[package]] 1595 + name = "sha2" 1596 + version = "0.11.1" 1597 + source = "registry+https://github.com/rust-lang/crates.io-index" 1598 + checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" 1599 + dependencies = [ 1600 + "cfg-if", 1601 + "cpufeatures", 1458 1602 "digest", 1459 1603 ] 1460 1604 ··· 1463 1607 version = "2.0.1" 1464 1608 source = "registry+https://github.com/rust-lang/crates.io-index" 1465 1609 checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" 1610 + 1611 + [[package]] 1612 + name = "signal-hook-registry" 1613 + version = "1.4.8" 1614 + source = "registry+https://github.com/rust-lang/crates.io-index" 1615 + checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" 1616 + dependencies = [ 1617 + "errno", 1618 + "libc", 1619 + ] 1466 1620 1467 1621 [[package]] 1468 1622 name = "simd-adler32" ··· 1608 1762 "libc", 1609 1763 "mio 1.2.4", 1610 1764 "pin-project-lite", 1765 + "signal-hook-registry", 1611 1766 "socket2", 1612 1767 "tokio-macros", 1613 1768 "windows-sys 0.61.2", ··· 1700 1855 checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" 1701 1856 dependencies = [ 1702 1857 "pin-project-lite", 1858 + "tracing-attributes", 1703 1859 "tracing-core", 1860 + ] 1861 + 1862 + [[package]] 1863 + name = "tracing-attributes" 1864 + version = "0.1.31" 1865 + source = "registry+https://github.com/rust-lang/crates.io-index" 1866 + checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" 1867 + dependencies = [ 1868 + "proc-macro2", 1869 + "quote", 1870 + "syn 2.0.119", 1704 1871 ] 1705 1872 1706 1873 [[package]] ··· 1744 1911 1745 1912 [[package]] 1746 1913 name = "untrusted" 1914 + version = "0.7.1" 1915 + source = "registry+https://github.com/rust-lang/crates.io-index" 1916 + checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" 1917 + 1918 + [[package]] 1919 + name = "untrusted" 1747 1920 version = "0.9.0" 1748 1921 source = "registry+https://github.com/rust-lang/crates.io-index" 1749 1922 checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" ··· 1765 1938 version = "1.0.4" 1766 1939 source = "registry+https://github.com/rust-lang/crates.io-index" 1767 1940 checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" 1768 - 1769 - [[package]] 1770 - name = "version_check" 1771 - version = "0.9.5" 1772 - source = "registry+https://github.com/rust-lang/crates.io-index" 1773 - checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" 1774 1941 1775 1942 [[package]] 1776 1943 name = "vm-memory"
+2 -2
tools/lazy-store/Cargo.toml
··· 13 13 log = "0.4" 14 14 serde = { version = "1", features = ["derive"] } 15 15 serde_json = "1" 16 - sha2 = "0.10" 17 16 tokio = { version = "1", features = ["rt-multi-thread", "macros", "fs", "io-util", "net", "time", "sync"] } 18 17 tokio-util = { version = "0.7", features = ["io", "io-util"] } 19 18 futures-util = "0.3" ··· 23 22 webpki-root-certs = "1" 24 23 async-compression = { version = "0.4", features = ["tokio", "zstd", "xz", "bzip2", "gzip", "brotli"] } 25 24 harmonia-file-core = { git = "https://github.com/nix-community/harmonia", rev = "3edb37ae36b561c2d395980170c70bbce292bbc5" } 26 - harmonia-utils-base-encoding = { git = "https://github.com/nix-community/harmonia", rev = "3edb37ae36b561c2d395980170c70bbce292bbc5" } 25 + harmonia-file-nar = { git = "https://github.com/nix-community/harmonia", rev = "3edb37ae36b561c2d395980170c70bbce292bbc5" } 26 + harmonia-utils-hash = { git = "https://github.com/nix-community/harmonia", rev = "3edb37ae36b561c2d395980170c70bbce292bbc5" } 27 27 28 28 [profile.release] 29 29 debug = 1
+37 -38
tools/lazy-store/src/lazy.rs
··· 21 21 ZeroCopyWriter, 22 22 }; 23 23 use futures_util::{stream, StreamExt}; 24 - use harmonia_utils_base_encoding::base32; 25 - use tokio_util::io::{InspectReader, SyncIoBridge}; 24 + use harmonia_file_core::{Directory, FileSystemObject, FileTree, Regular, Symlink}; 25 + use harmonia_utils_hash::HashFormat; 26 + use tokio_util::io::InspectReader; 26 27 27 28 use crate::closure::PathInfo; 28 29 use crate::http::{self, Http}; 29 - use crate::nar::{self, Tree}; 30 + use crate::nar::{self, Size, Tree}; 30 31 use crate::stats::Stats; 31 32 use crate::Backing; 32 33 ··· 179 180 match self.http.get(&format!("{}/{h}.ls", info.cache)).await { 180 181 Ok((body, len)) => { 181 182 self.stats.add("ls", len as u64); 182 - Tree::from_listing(&body) 183 + nar::from_listing(&body) 183 184 } 184 185 // no listing: fall back to the NAR 185 - Err(http::Error::NotFound) => Tree::from_path(&self.fetch(i).await?), 186 + Err(http::Error::NotFound) => nar::from_path(&self.fetch(i).await?), 186 187 Err(e) => Err(e.into()), 187 188 } 188 189 } 189 190 190 191 /// Add the nodes of `t` (at `rel` in store path `path`), return its inode. 191 - fn add(&mut self, path: u32, rel: String, t: Tree) -> u32 { 192 + fn add(&mut self, path: u32, rel: String, FileTree(t): Tree) -> u32 { 192 193 let ino = self.nodes.len() as u32 + 1; 193 194 let (kind, entries) = match t { 194 - Tree::File { size, exec } => (Kind::File { size, exec }, None), 195 - Tree::Symlink(s) => (Kind::Symlink(s.into_bytes().into()), None), 196 - Tree::Dir(e) => (Kind::Dir(Vec::new()), Some(e)), 195 + FileSystemObject::Regular(r) => { 196 + let (size, exec) = (r.contents.size, r.executable); 197 + (Kind::File { size, exec }, None) 198 + } 199 + FileSystemObject::Symlink(s) => (Kind::Symlink(s.target.into_bytes().into()), None), 200 + FileSystemObject::Directory(d) => (Kind::Dir(Vec::new()), Some(d.entries)), 197 201 }; 198 202 self.nodes.push(Node { 199 203 kind, ··· 208 212 } else { 209 213 format!("{rel}/{name}") 210 214 }; 211 - children.push((name.into_boxed_str(), self.add(path, r, c))); 215 + children.push((name.into_boxed_str(), self.add(path, r, *c))); 212 216 } 213 217 // BTreeMap order is sorted already 214 218 self.nodes[ino as usize - 1].kind = Kind::Dir(children); ··· 218 222 219 223 /// The listing of `ino` as built from the `.ls` (for `fetch`). 220 224 fn tree_of(&self, ino: u64) -> Tree { 221 - match &self.nodes[ino as usize - 1].kind { 222 - Kind::Dir(c) => Tree::Dir( 223 - c.iter() 224 - .map(|(n, i)| (n.to_string(), self.tree_of(*i as u64))) 225 + FileTree(match &self.nodes[ino as usize - 1].kind { 226 + Kind::Dir(c) => FileSystemObject::Directory(Directory { 227 + entries: c 228 + .iter() 229 + .map(|(n, i)| (n.to_string(), Box::new(self.tree_of(*i as u64)))) 225 230 .collect(), 226 - ), 227 - Kind::File { size, exec } => Tree::File { 228 - size: *size, 229 - exec: *exec, 230 - }, 231 - Kind::Symlink(t) => Tree::Symlink(String::from_utf8_lossy(t).into_owned()), 232 - } 231 + }), 232 + Kind::File { size, exec } => FileSystemObject::Regular(Regular { 233 + executable: *exec, 234 + contents: Size { size: *size }, 235 + }), 236 + Kind::Symlink(t) => FileSystemObject::Symlink(Symlink { 237 + target: String::from_utf8_lossy(t).into_owned(), 238 + }), 239 + }) 233 240 } 234 241 235 242 /// Fetch store path `name` as an open() would and check that what got ··· 240 247 let p = self 241 248 .fetch(self.nodes[ino as usize - 1].path as usize) 242 249 .await?; 243 - if Tree::from_path(&p)? != self.tree_of(ino) { 250 + if nar::from_path(&p)? != self.tree_of(ino) { 244 251 return Err(io::Error::other(format!( 245 252 "{name}: unpacked tree differs from its listing" 246 253 ))); ··· 357 364 .open(&format!("{}/{}", info.cache, info.url)) 358 365 .await?; 359 366 // the compressed download size 360 - let file_size = Arc::new(AtomicU64::new(0)); 361 - let counted = InspectReader::new(body, { 362 - let n = file_size.clone(); 363 - move |b: &[u8]| { 364 - n.fetch_add(b.len() as u64, Ordering::Relaxed); 365 - } 366 - }); 367 - let nar = SyncIoBridge::new(http::decode_body(&info.compression, counted)?); 368 - let unpack_to = tmp.clone(); 369 - let (digest, size) = tokio::task::spawn_blocking(move || nar::unpack(nar, &unpack_to)) 370 - .await 371 - .map_err(io::Error::other)??; 372 - let file_size = file_size.load(Ordering::Relaxed); 373 - if !nar::hash_matches(&info.nar_hash, &digest) || size != info.nar_size { 367 + let mut file_size = 0; 368 + let mut body = InspectReader::new(body, |b: &[u8]| file_size += b.len() as u64); 369 + let nar = http::decode_body(&info.compression, &mut body)?; 370 + let (hash, size) = nar::unpack(nar, &tmp).await?; 371 + drop(body); 372 + if !nar::hash_matches(&info.nar_hash, &hash) || size != info.nar_size { 374 373 nar::remove(&tmp)?; 375 374 return Err(io::Error::other(format!( 376 - "{name}: NAR hash mismatch, sha256:{} ({size} bytes) != {} ({} bytes)", 377 - base32::encode_string(&digest), 375 + "{name}: NAR hash mismatch, {} ({size} bytes) != {} ({} bytes)", 376 + hash.as_base32(), 378 377 info.nar_hash, 379 378 info.nar_size 380 379 )));
+109 -277
tools/lazy-store/src/nar.rs
··· 2 2 3 3 use std::collections::BTreeMap; 4 4 use std::fs; 5 - use std::io::{self, Read, Write}; 6 - use std::os::unix::ffi::OsStrExt; 7 - use std::os::unix::fs::{symlink, MetadataExt, PermissionsExt}; 5 + use std::io; 6 + use std::os::unix::fs::{MetadataExt, PermissionsExt}; 8 7 use std::path::Path; 9 8 10 - use harmonia_file_core::{FileSystemObject, FileTree}; 11 - use harmonia_utils_base_encoding::base32; 9 + use futures_util::TryStreamExt; 10 + use harmonia_file_core::{Directory, FileSystemObject, FileTree, Regular, Symlink}; 11 + use harmonia_file_nar::{parse_nar, restore, NarWriteError}; 12 + use harmonia_utils_hash::{fmt::Any, Algorithm, Context, Hash}; 12 13 use serde::Deserialize; 13 - use sha2::{Digest, Sha256}; 14 + use tokio::io::{AsyncRead, AsyncReadExt}; 15 + use tokio_util::io::InspectReader; 14 16 15 - /// Does `digest` match a narinfo NarHash (`sha256:<nix32>` or base16)? 16 - pub fn hash_matches(expected: &str, digest: &[u8]) -> bool { 17 - let Some(h) = expected.strip_prefix("sha256:") else { 18 - return false; 19 - }; 20 - let hex: String = digest.iter().map(|b| format!("{b:02x}")).collect(); 21 - h == base32::encode_string(digest) || h == hex 17 + /// Does `actual` match a narinfo NarHash (`sha256:` in any encoding)? 18 + pub fn hash_matches(expected: &str, actual: &Hash) -> bool { 19 + expected 20 + .parse::<Any<Hash>>() 21 + .is_ok_and(|e| e.into_hash() == *actual) 22 + } 23 + 24 + #[derive(Debug, Clone, PartialEq, Deserialize)] 25 + pub struct Size { 26 + pub size: u64, 22 27 } 23 28 24 29 /// The metadata of a store path, as in the caches' `.ls` listings. 25 - #[derive(Debug, Clone, PartialEq)] 26 - pub enum Tree { 27 - Dir(BTreeMap<String, Tree>), 28 - File { size: u64, exec: bool }, 29 - Symlink(String), 30 + pub type Tree = FileTree<Size>; 31 + 32 + /// Parse a `.ls` listing (`{"version": 1, "root": {...}}`). 33 + pub fn from_listing(body: &[u8]) -> io::Result<Tree> { 34 + #[derive(Deserialize)] 35 + struct Listing { 36 + root: Tree, 37 + } 38 + let l: Listing = serde_json::from_slice(body).map_err(io::Error::other)?; 39 + Ok(l.root) 30 40 } 31 41 32 - impl Tree { 33 - /// Parse a `.ls` listing (`{"version": 1, "root": {...}}`). 34 - pub fn from_listing(body: &[u8]) -> io::Result<Tree> { 35 - #[derive(Deserialize)] 36 - struct Size { 37 - size: u64, 42 + /// Listing of an unpacked store path (for paths without `.ls`). 43 + pub fn from_path(p: &Path) -> io::Result<Tree> { 44 + let m = fs::symlink_metadata(p)?; 45 + let t = m.file_type(); 46 + Ok(FileTree(if t.is_dir() { 47 + let mut entries = BTreeMap::new(); 48 + for d in fs::read_dir(p)? { 49 + let d = d?; 50 + let name = d.file_name().into_string().map_err(|n| { 51 + io::Error::other(format!("non-UTF-8 name {n:?} in {}", p.display())) 52 + })?; 53 + entries.insert(name, Box::new(from_path(&d.path())?)); 38 54 } 39 - #[derive(Deserialize)] 40 - struct Listing { 41 - root: FileTree<Size>, 42 - } 43 - fn tree(FileTree(t): FileTree<Size>) -> Tree { 44 - match t { 45 - FileSystemObject::Directory(d) => { 46 - Tree::Dir(d.entries.into_iter().map(|(k, c)| (k, tree(*c))).collect()) 47 - } 48 - FileSystemObject::Regular(r) => Tree::File { 49 - size: r.contents.size, 50 - exec: r.executable, 51 - }, 52 - FileSystemObject::Symlink(s) => Tree::Symlink(s.target), 53 - } 54 - } 55 - let l: Listing = serde_json::from_slice(body).map_err(io::Error::other)?; 56 - Ok(tree(l.root)) 57 - } 58 - 59 - /// Listing of an unpacked store path (for paths without `.ls`). 60 - pub fn from_path(p: &Path) -> io::Result<Tree> { 61 - let m = fs::symlink_metadata(p)?; 62 - let t = m.file_type(); 63 - Ok(if t.is_dir() { 64 - let mut e = BTreeMap::new(); 65 - for d in fs::read_dir(p)? { 66 - let d = d?; 67 - let name = d.file_name().into_string().map_err(|n| { 68 - io::Error::other(format!("non-UTF-8 name {n:?} in {}", p.display())) 69 - })?; 70 - e.insert(name, Tree::from_path(&d.path())?); 71 - } 72 - Tree::Dir(e) 73 - } else if t.is_symlink() { 74 - Tree::Symlink(fs::read_link(p)?.to_string_lossy().into_owned()) 75 - } else { 76 - Tree::File { 77 - size: m.size(), 78 - exec: m.mode() & 0o100 != 0, 79 - } 55 + FileSystemObject::Directory(Directory { entries }) 56 + } else if t.is_symlink() { 57 + let target = fs::read_link(p)?.to_string_lossy().into_owned(); 58 + FileSystemObject::Symlink(Symlink { target }) 59 + } else { 60 + FileSystemObject::Regular(Regular { 61 + executable: m.mode() & 0o100 != 0, 62 + contents: Size { size: m.size() }, 80 63 }) 81 - } 64 + })) 82 65 } 83 66 84 - /// Reads through to `inner`, hashing and counting what passes. 85 - pub struct HashingReader<R> { 86 - inner: R, 87 - pub hasher: Sha256, 88 - pub n: u64, 67 + /// Unpack the NAR read from `r` to `dest` (which must not exist), read-only 68 + /// like in the store. Returns the NAR's sha256 and size; checking them is 69 + /// up to the caller. Data after the NAR counts into the size. 70 + pub async fn unpack(r: impl AsyncRead + Unpin + Send, dest: &Path) -> io::Result<(Hash, u64)> { 71 + let mut hash = Context::new(Algorithm::SHA256); 72 + let mut size = 0; 73 + let mut r = InspectReader::new(r, |b: &[u8]| { 74 + hash.update(b); 75 + size += b.len() as u64; 76 + }); 77 + let events = 78 + parse_nar(&mut r).map_err(|e| NarWriteError::create_file_error(dest.to_owned(), e)); 79 + restore(events, dest).await.map_err(io::Error::other)?; 80 + r.read(&mut [0; 1]).await?; 81 + drop(r); 82 + let d = dest.to_owned(); 83 + tokio::task::spawn_blocking(move || read_only(&d)) 84 + .await 85 + .map_err(io::Error::other)??; 86 + Ok((hash.finish(), size)) 89 87 } 90 88 91 - impl<R: Read> Read for HashingReader<R> { 92 - fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { 93 - let n = self.inner.read(buf)?; 94 - self.hasher.update(&buf[..n]); 95 - self.n += n as u64; 96 - Ok(n) 97 - } 98 - } 99 - 100 - fn bad(msg: impl Into<String>) -> io::Error { 101 - io::Error::new(io::ErrorKind::InvalidData, format!("NAR: {}", msg.into())) 102 - } 103 - 104 - struct Nar<R> { 105 - r: HashingReader<R>, 106 - } 107 - 108 - impl<R: Read> Nar<R> { 109 - fn int(&mut self) -> io::Result<u64> { 110 - let mut b = [0u8; 8]; 111 - self.r.read_exact(&mut b)?; 112 - Ok(u64::from_le_bytes(b)) 113 - } 114 - 115 - fn pad(&mut self, n: u64) -> io::Result<()> { 116 - let p = ((8 - n % 8) % 8) as usize; 117 - let mut b = [0u8; 8]; 118 - self.r.read_exact(&mut b[..p])?; 119 - if b[..p].iter().any(|&x| x != 0) { 120 - return Err(bad("non-zero padding")); 89 + /// Store permissions: 0444 or 0555 for files, 0555 for directories. 90 + fn read_only(p: &Path) -> io::Result<()> { 91 + let m = fs::symlink_metadata(p)?; 92 + let mode = if m.is_symlink() { 93 + return Ok(()); 94 + } else if m.is_dir() { 95 + for d in fs::read_dir(p)? { 96 + read_only(&d?.path())?; 121 97 } 122 - Ok(()) 123 - } 124 - 125 - fn str(&mut self, limit: u64) -> io::Result<Vec<u8>> { 126 - let n = self.int()?; 127 - if n > limit { 128 - return Err(bad("string too long")); 129 - } 130 - let mut s = vec![0u8; n as usize]; 131 - self.r.read_exact(&mut s)?; 132 - self.pad(n)?; 133 - Ok(s) 134 - } 135 - 136 - fn expect(&mut self, s: &[u8]) -> io::Result<()> { 137 - let got = self.str(4096)?; 138 - if got != s { 139 - return Err(bad(format!( 140 - "expected {:?}, got {:?}", 141 - String::from_utf8_lossy(s), 142 - String::from_utf8_lossy(&got) 143 - ))); 144 - } 145 - Ok(()) 146 - } 147 - 148 - fn node(&mut self, dest: &Path) -> io::Result<()> { 149 - self.expect(b"(")?; 150 - self.expect(b"type")?; 151 - match &self.str(4096)?[..] { 152 - b"regular" => { 153 - let mut tag = self.str(4096)?; 154 - let mut mode = 0o444; 155 - if tag == b"executable" { 156 - self.expect(b"")?; 157 - tag = self.str(4096)?; 158 - mode = 0o555; 159 - } 160 - if tag != b"contents" { 161 - return Err(bad("expected contents")); 162 - } 163 - let n = self.int()?; 164 - let mut out = io::BufWriter::with_capacity(1 << 20, fs::File::create(dest)?); 165 - let copied = io::copy(&mut (&mut self.r).take(n), &mut out)?; 166 - if copied != n { 167 - return Err(bad("short file contents")); 168 - } 169 - out.flush()?; 170 - self.pad(n)?; 171 - fs::set_permissions(dest, fs::Permissions::from_mode(mode))?; 172 - self.expect(b")")?; 173 - } 174 - b"symlink" => { 175 - self.expect(b"target")?; 176 - let t = self.str(4096)?; 177 - symlink(std::ffi::OsStr::from_bytes(&t), dest)?; 178 - self.expect(b")")?; 179 - } 180 - b"directory" => { 181 - fs::create_dir(dest)?; 182 - loop { 183 - let tag = self.str(4096)?; 184 - if tag == b")" { 185 - break; 186 - } 187 - if tag != b"entry" { 188 - return Err(bad("expected entry")); 189 - } 190 - self.expect(b"(")?; 191 - self.expect(b"name")?; 192 - let name = self.str(255)?; 193 - if name.is_empty() || name == b"." || name == b".." || name.contains(&b'/') { 194 - return Err(bad("bad entry name")); 195 - } 196 - self.expect(b"node")?; 197 - let name = String::from_utf8(name).map_err(|_| bad("non-UTF-8 name"))?; 198 - self.node(&dest.join(name))?; 199 - self.expect(b")")?; 200 - } 201 - fs::set_permissions(dest, fs::Permissions::from_mode(0o555))?; 202 - } 203 - t => { 204 - return Err(bad(format!( 205 - "unknown type {:?}", 206 - String::from_utf8_lossy(t) 207 - ))) 208 - } 209 - } 210 - Ok(()) 211 - } 212 - } 213 - 214 - /// Unpack the NAR read from `r` to `dest` (which must not exist). Returns 215 - /// the NAR's sha256 and size; checking them is up to the caller. Fails if 216 - /// anything follows the NAR. 217 - pub fn unpack(r: impl Read, dest: &Path) -> io::Result<(Vec<u8>, u64)> { 218 - let mut nar = Nar { 219 - r: HashingReader { 220 - inner: r, 221 - hasher: Sha256::new(), 222 - n: 0, 223 - }, 98 + 0o555 99 + } else if m.mode() & 0o100 != 0 { 100 + 0o555 101 + } else { 102 + 0o444 224 103 }; 225 - nar.expect(b"nix-archive-1")?; 226 - nar.node(dest)?; 227 - let mut rest = [0u8; 1]; 228 - if nar.r.inner.read(&mut rest)? != 0 { 229 - return Err(bad("trailing data")); 230 - } 231 - let n = nar.r.n; 232 - Ok((nar.r.hasher.finalize().to_vec(), n)) 104 + fs::set_permissions(p, fs::Permissions::from_mode(mode)) 233 105 } 234 106 235 107 /// Remove a (partially) unpacked tree, including read-only directories. ··· 254 126 mod tests { 255 127 use super::*; 256 128 257 - fn s(out: &mut Vec<u8>, b: &[u8]) { 258 - out.extend_from_slice(&(b.len() as u64).to_le_bytes()); 259 - out.extend_from_slice(b); 260 - out.resize(out.len() + (8 - b.len() % 8) % 8, 0); 261 - } 262 - 263 - fn nar() -> Vec<u8> { 264 - let mut o = Vec::new(); 265 - for t in [&b"nix-archive-1"[..], b"(", b"type", b"directory"] { 266 - s(&mut o, t); 267 - } 268 - for t in [&b"entry"[..], b"(", b"name", b"bin", b"node", b"("] { 269 - s(&mut o, t); 270 - } 271 - for t in [ 272 - &b"type"[..], 273 - b"regular", 274 - b"executable", 275 - b"", 276 - b"contents", 277 - b"#!/bin/sh\n", 278 - ] { 279 - s(&mut o, t); 280 - } 281 - for t in [&b")"[..], b")"] { 282 - s(&mut o, t); 283 - } 284 - for t in [&b"entry"[..], b"(", b"name", b"link", b"node", b"("] { 285 - s(&mut o, t); 286 - } 287 - for t in [ 288 - &b"type"[..], 289 - b"symlink", 290 - b"target", 291 - b"bin", 292 - b")", 293 - b")", 294 - b")", 295 - ] { 296 - s(&mut o, t); 297 - } 298 - o 299 - } 300 - 301 129 #[test] 302 130 fn hash_matches_nix() { 303 131 // nix hash convert --to nix32 sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= 304 - let d = Sha256::digest(b""); 132 + let d = Algorithm::SHA256.digest(b""); 305 133 assert!(hash_matches( 306 134 "sha256:0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73", 307 135 &d ··· 316 144 )); 317 145 } 318 146 319 - #[test] 320 - fn unpack_and_list() { 321 - let dir = std::env::temp_dir().join(format!("lazy-store-test-{}", std::process::id())); 322 - let _ = remove(&dir); 323 - let data = nar(); 324 - let (h, n) = unpack(&data[..], &dir).unwrap(); 147 + #[tokio::test] 148 + async fn unpack_and_list() { 149 + let tmp = std::env::temp_dir().join(format!("lazy-store-test-{}", std::process::id())); 150 + let (src, dir) = (tmp.join("src"), tmp.join("out")); 151 + let _ = remove(&tmp); 152 + fs::create_dir_all(&src).unwrap(); 153 + fs::write(src.join("bin"), "#!/bin/sh\n").unwrap(); 154 + fs::set_permissions(src.join("bin"), fs::Permissions::from_mode(0o755)).unwrap(); 155 + std::os::unix::fs::symlink("bin", src.join("link")).unwrap(); 156 + let data: Vec<u8> = harmonia_file_nar::NarByteStream::new(src.clone()) 157 + .try_collect::<Vec<_>>() 158 + .await 159 + .unwrap() 160 + .concat(); 161 + 162 + let (h, n) = unpack(&data[..], &dir).await.unwrap(); 325 163 assert_eq!(n, data.len() as u64); 326 - assert_eq!(h, Sha256::digest(&data).to_vec()); 327 - let t = Tree::from_path(&dir).unwrap(); 328 - let mut e = BTreeMap::new(); 329 - e.insert( 330 - "bin".into(), 331 - Tree::File { 332 - size: 10, 333 - exec: true, 334 - }, 335 - ); 336 - e.insert("link".into(), Tree::Symlink("bin".into())); 337 - assert_eq!(t, Tree::Dir(e)); 164 + assert_eq!(h, Algorithm::SHA256.digest(&data)); 165 + assert_eq!(fs::metadata(dir.join("bin")).unwrap().mode() & 0o777, 0o555); 166 + assert_eq!(fs::metadata(&dir).unwrap().mode() & 0o777, 0o555); 167 + let t = from_path(&dir).unwrap(); 338 168 let ls = br#"{"version":1,"root":{"type":"directory","entries":{ 339 169 "bin":{"type":"regular","size":10,"executable":true,"narOffset":96}, 340 170 "link":{"type":"symlink","target":"bin"}}}}"#; 341 - assert_eq!(Tree::from_listing(ls).unwrap(), t); 171 + assert_eq!(from_listing(ls).unwrap(), t); 342 172 remove(&dir).unwrap(); 343 173 174 + // the caller's NarSize check rejects it 344 175 let mut trailing = data.clone(); 345 176 trailing.push(0); 346 - assert!(unpack(&trailing[..], &dir).is_err()); 347 - remove(&dir).unwrap(); 177 + let (_, n) = unpack(&trailing[..], &dir).await.unwrap(); 178 + assert_eq!(n, trailing.len() as u64); 179 + remove(&tmp).unwrap(); 348 180 } 349 181 }